:W&Bá¾Cam M j1qd7654ª; 9 /0.28.2+dc1ef8bec7642bea1cb89c3f6f8fc64f536def4c0.70.0dÐy¯Î‚" 0hhlaac3t58f j1qd7654Šª j1qd7654spatop" _wandb‚{}Bspatop_legacyfr_fixed_outputj$dp-spatop-train-legacyfr-fixed-mf4xjŠ ÚþøÓ¸ÉçÝš1 (H¨¸h"3.12.10*0.28.2j linux-x86_64`4´ßeÒß -Linux-6.8.0-134-generic-x86_64-with-glibc2.36CPython 3.12.10 ÚþøÓÀܘ*-of*9/data/spatop/options_files/v11/spatop_legacyfr_fixed.json*-l*/scratch/logs/*-n*spatop_legacyfr_fixed_output2-m spanet.trainRdanielprimosch@gmail.comZ/scratch/logs/b$dp-spatop-train-legacyfr-fixed-mf4xjr/opt/conda/bin/pythonº m61lprc7afplhtmjkygw5e5tomd6cratå"áÉÒÀ€ˆ€’NVIDIA A100-SXM4-80GB˜¢ /€€€¥È.€ ‡‹ ª€àçÓ½ÂS NVIDIA A100-SXM4-80GB€€€€À€6"Ampere*(GPU-64b3c6a5-1109-a494-c90a-262e81936bd2Ê13.2º m61lprc7afplhtmjkygw5e5tomd6cratæPò2  requirements.txt3Ç&P j1qd7654Z? (H¨¸ (H¨¸h"3.12.10*0.28.2j linux-x86_64Ù@8—R j1qd7654ZA (H¨¸ (H¨¸8h"3.12.10*0.28.2j linux-x86_64ö„œ&* j1qd7654b trainer/global_step2œðU  j1qd7654ZD (H¨¸ (H¨¸8h"3.12.10*0.28.2j linux-x86_64߸^ /  j1qd7654b*"trainer/global_step2¯§˜qK  j1qd7654j: ÛþøÓøÿ¼‹*LOCAL_RANK: 0 - CUDA_VISIBLE_DEVICES: [0] éØ«s“  j1qd7654j  ÛþøÓ°ú¼ªî â”â”â”â”┳â”â”â”â”â”â”â”â”â”â”â”â”â”â”â”â”â”â”â”â”â”â”â”â”┳â”â”â”â”â”â”â”â”â”â”â”â”â”â”â”â”â”â”â”â”â”â”â”â”┳â”â”â”â”â”â”â”â”┳â”â”â”â”â”â”â”┳â”â”â”â”â”â”â”┓ ┃ ┃ Name ┃ Type ┃ Params ┃ Mode ┃ FLOPs ┃ ┡â”â”â”╇â”â”â”â”â”â”â”â”â”â”â”â”â”â”â”â”â”â”â”â”â”â”â”â”╇â”â”â”â”â”â”â”â”â”â”â”â”â”â”â”â”â”â”â”â”â”â”â”â”╇â”â”â”â”â”â”â”â”╇â”â”â”â”â”â”â”╇â”â”â”â”â”â”â”┩ │ 0 │ embedding │ MultiInputVectorEmbed… │ 105 K │ train │ 0 │ │ 1 │ encoder │ JetEncoder │ 498 K │ train │ 0 │ │ 2 │ branch_decoders │ ModuleList │ 1.9 M │ train │ 0 │ │ 3 │ regression_decoder │ RegressionDecoder │ 0 │ train │ 0 │ │ 4 │ classification_decoder │ ClassificationDecoder │ 0 │ train │ 0 │ │ │ other params │ n/a │ 10 │ n/a │ n/a │ └───┴────────────────────────┴────────────────────────┴────────┴───────┴───────┘ >Ôd,\  j1qd7654jÊ ÛþøÓØÊ«·Trainable params: 2.5 M Non-trainable params: 22 Total params: 2.5 M Total estimated model params size (MB): 10.170 Modules in train mode: 1101 Modules in eval mode: 0 Total FLOPs: 0 .’?\ j1qd7654*Ê  hidden_dim‚64  transformer_dim‚64  transformer_dim_scale‚2.0  initial_embedding_dim‚16  position_embedding_dim‚32  num_embedding_layers‚10  num_encoder_layers‚6 ! num_branch_embedding_layers‚3  num_branch_encoder_layers‚3  num_jet_embedding_layers‚0  num_jet_encoder_layers‚2  num_detector_layers‚2  num_regression_layers‚3  num_classification_layers‚3  split_symmetric_attention‚1  num_attention_heads‚4 ! transformer_activation‚"gelu"  skip_connections‚1 ( "initial_embedding_skip_connections‚1  linear_block_type‚"GRU"  transformer_type‚"Gated"  linear_activation‚"gelu"  normalization‚ "LayerNorm"  masking‚ "Filling" linear_prelu_activation‚true # use_pairwise_interactions‚false  num_pairwise_features‚4  pairwise_embedding_dim‚8  pairwise_input_source‚""  use_moe‚false  num_experts‚8  num_experts_per_tok‚2  moe_loss_scale‚0.01 K event_info_file‚7"/data/spatop/event_files/v11/tt_hadronic_fr_only.yaml" F training_file‚4"/data/spatop/tpm70_wpm30_FB350_fixed/all_merged.h5"  validation_file‚""  testing_file‚""  normalize_features‚1  limit_to_num_jets‚0  balance_particles‚1  balance_jets‚0 ! balance_classifications‚false  partial_events‚1  dataset_limit‚1.0  dataset_randomization‚0  train_validation_split‚0.95  limit_val_batches‚1.0  batch_size‚2048  num_dataloader_workers‚0  mask_sequence_vectors‚1  combine_pair_loss‚"min"  optimizer‚"AdamW"  learning_rate‚0.0015  focal_gamma‚0.0  combinatorial_scale‚0.0 # learning_rate_warmup_epochs‚1.0  learning_rate_cycles‚1  assignment_loss_scale‚1.0  detection_loss_scale‚1.0  kl_loss_scale‚0.0  regression_loss_scale‚0.0 ! classification_loss_scale‚0.0 ! use_mass_classification‚false  mass_classes‚[]  balance_losses‚true  l2_penalty‚0.0002  gradient_clip‚0.0  dropout‚0.0 epochs‚50 num_gpu‚1  verbose_output‚false  usable_gpus‚""  trial_time‚"" $ trial_output_dir‚"./test_output"_ _?ë j1qd7654jÙ ÛþøÓП¯®È/opt/conda/lib/python3.12/site-packages/pytorch_lightning/utilities/_pytree.py:21: `isinstance(treespec, LeafSpec)` is deprecated, use `isinstance(treespec, TreeSpec) and treespec.is_leaf()` instead. …jN j1qd7654j¼ ÛþøÓÀ¡¾®«/opt/conda/lib/python3.12/site-packages/pytorch_lightning/trainer/connectors/data_connector.py:434: The 'val_dataloader' does not have many workers which may be a bottleneck. Consider increasing the value of the `num_workers` argument` to `num_workers=3` in the `DataLoader` to improve performance. PLxê j1qd7654jØ ÝþøÓП;È/opt/conda/lib/python3.12/site-packages/pytorch_lightning/utilities/_pytree.py:21: `isinstance(treespec, LeafSpec)` is deprecated, use `isinstance(treespec, TreeSpec) and treespec.is_leaf()` instead. Ÿ"> O j1qd7654j½ ÝþøÓ°Á®;­/opt/conda/lib/python3.12/site-packages/pytorch_lightning/trainer/connectors/data_connector.py:434: The 'train_dataloader' does not have many workers which may be a bottleneck. Consider increasing the value of the `num_workers` argument` to `num_workers=3` in the `DataLoader` to improve performance. DªšS“‚(:‰ êþøÓÇÏŠá gpu.0.gpu‚34.0 gpu.0.memory‚8, gpu.0.memoryAllocated‚7.0288848876953125) gpu.0.memoryAllocatedBytes‚ 6037766144 gpu.0.temp‚30.0 gpu.0.powerWatts‚117.415' gpu.0.enforcedPowerLimitWatts‚500.0 gpu.0.powerPercent‚23.483 gpu.0.smClock‚1410 gpu.0.memoryClock‚1593$ gpu.0.correctedMemoryErrors‚1955# gpu.0.uncorrectedMemoryErrors‚0œ6 f‚(:Ü êþøÓª€Üæ) proc.memory.availableMB‚ 6105.39453125 proc.cpu.threads‚17 disk.sda1.out‚0 disk.dm-0.in‚ 7.5546875 disk.sda3.in‚ 161.1640625 cpu‚18.167842489630992$ disk./.usageGB‚456.0214157104492 disk.sda.in‚ 161.1640625 disk.dm-1.in‚ 161.1640625 disk.sda3.out‚0.53125 disk.nvme0c0n1.in‚ 7.5546875% disk.nvme0c0n1.out‚43.86181640625! proc.memory.rssMB‚ 2251.421875* proc.memory.percent‚13.741588592529297 network.recv‚89447* disk./.usagePercent‚30.603979835781615 disk.sda1.in‚0 disk.sda2.in‚0 disk.nvme0n1.in‚ 7.5546875 disk.drbd2165.in‚0 disk.drbd2165.out‚0% memory_percent‚62.735629081726074 network.sent‚82529 disk.sda.out‚0.53125 disk.sda2.out‚0# disk.nvme0n1.out‚43.86181640625 disk.dm-1.out‚0.53125 disk.dm-0.out‚43.86181640625%¬Ä93b/ lr-AdamW/pg1"trainer/global_step2:JP-s`3b/ lr-AdamW/pg2"trainer/global_step2:JPU˜*«-) _wandbruntime‚15  _runtime‚15jh ª¥ trainer/global_step‚49 % _timestamp‚1.7866586670438173e+09 _runtime‚ 15.573985922 _step‚0  lr-AdamW/pg1‚ 0.00018375  lr-AdamW/pg2‚ 0.00018375rÝí(¬§ % _timestamp‚1.7866586670438173e+09 _runtime‚ 15.573985922 _step‚0  lr-AdamW/pg1‚ 0.00018375  lr-AdamW/pg2‚ 0.00018375 trainer/global_step‚49xp(Y_b[ 8DeviceStatsMonitor.on_train_batch_start/active.all.freed"trainer/global_step2:JPCïÿ–sbo LDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak"trainer/global_step2:JPd„>tlbh EDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated"trainer/global_step2:JP8”çqbm JDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current"trainer/global_step2:JP0dJÃmbi FDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak"trainer/global_step2:JP.NgÍab] :DeviceStatsMonitor.on_train_batch_start/active.all.current"trainer/global_step2:JPAAÐh bd ADeviceStatsMonitor.on_train_batch_start/active.large_pool.current"trainer/global_step2:JP£±?€o!bk HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed"trainer/global_step2:JPiئÓs"bo LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated"trainer/global_step2:JP;·l^#bZ 7DeviceStatsMonitor.on_train_batch_start/num_device_free"trainer/global_step2:JP[ÑJ§i$be BDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current"trainer/global_step2:JPuŽ#s%bo LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated"trainer/global_step2:JPSŽ+=i&be BDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated"trainer/global_step2:JPDŇ”i'be BDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak"trainer/global_step2:JPwµv(br ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current"trainer/global_step2:JPOM–r)bn KDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated"trainer/global_step2:JPw@¾“j*bf CDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed"trainer/global_step2:JPKFúc+b_ DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak"trainer/global_step2:JP”oWól2bh EDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated"trainer/global_step2:JP\e7Ôs3bo LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated"trainer/global_step2:JP΄ [n4bj GDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak"trainer/global_step2:JPëD‘q5bm JDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current"trainer/global_step2:JPí ½Éa6b] :DeviceStatsMonitor.on_train_batch_start/num_oom_rejections"trainer/global_step2:JPO£Ý]j7bf CDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current"trainer/global_step2:JPQžWf8bb ?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak"trainer/global_step2:JPÿrsºn9bj GDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated"trainer/global_step2:JPlÏ;Ho:bk HDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current"trainer/global_step2:JP¹zÌìj;bf CDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated"trainer/global_step2:JP‹êY¬lbg DDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak"trainer/global_step2:JP>ôÚÈg?bc @DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak"trainer/global_step2:JPK¹Làg@bc @DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak"trainer/global_step2:JP™ úœ`Ab\ 9DeviceStatsMonitor.on_train_batch_start/segment.all.freed"trainer/global_step2:JPheÔ nBbj GDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed"trainer/global_step2:JPÞ&AüsCbo LDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak"trainer/global_step2:JP0l³¿hDbd ADeviceStatsMonitor.on_train_batch_start/oversize_segments.current"trainer/global_step2:JP«æ5 hEbd ADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed"trainer/global_step2:JP:7boFbk HDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed"trainer/global_step2:JPNBê*kGbg DDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated"trainer/global_step2:JP©ç^HbZ 7DeviceStatsMonitor.on_train_batch_start/active.all.peak"trainer/global_step2:JPÓ‰ÈÈfIbb ?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed"trainer/global_step2:JPQB¦ jJbf CDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated"trainer/global_step2:JP‘7vmnKbj GDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak"trainer/global_step2:JP™¸Þ kLbg DDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated"trainer/global_step2:JPç[?giMbe BDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current"trainer/global_step2:JP•Gª…tNbp MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed"trainer/global_step2:JP÷âIÆkObg DDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak"trainer/global_step2:JPƒÅ~qPbm JDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current"trainer/global_step2:JPÃ?||vQbr ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current"trainer/global_step2:JPF€§fRbb ?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed"trainer/global_step2:JPögŽpSbl IDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current"trainer/global_step2:JP»‡ÆèdTb` =DeviceStatsMonitor.on_train_batch_start/segment.all.allocated"trainer/global_step2:JP„HÔ&rUbn KDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated"trainer/global_step2:JP²GéiVbe BDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed"trainer/global_step2:JP ¤ÍÙeWba >DeviceStatsMonitor.on_train_batch_start/allocation.all.current"trainer/global_step2:JPT'Ä2iXbe BDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak"trainer/global_step2:JP”ÑÔrYbn KDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated"trainer/global_step2:JPâû¡_Zb[ 8DeviceStatsMonitor.on_train_batch_start/num_device_alloc"trainer/global_step2:JPPRb[b^ ;DeviceStatsMonitor.on_train_batch_start/segment.all.current"trainer/global_step2:JPÖ³–„i\be BDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current"trainer/global_step2:JPejÄn]bj GDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated"trainer/global_step2:JP'Ï;;h^bd ADeviceStatsMonitor.on_train_batch_start/active.small_pool.current"trainer/global_step2:JPõJÄ"t_bp MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed"trainer/global_step2:JP&3b p`bl IDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current"trainer/global_step2:JPáÂpabl IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated"trainer/global_step2:JP6,¡jbbf CDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current"trainer/global_step2:JPá‘5Jpcbl IDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current"trainer/global_step2:JPÚßDmdbi FDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed"trainer/global_step2:JP. |mebi FDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak"trainer/global_step2:JPc­ffbb ?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed"trainer/global_step2:JPæÍv9ggbc @DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current"trainer/global_step2:JP»¹Qohbk HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed"trainer/global_step2:JPúêðkibg DDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated"trainer/global_step2:JP—µÛ»fjbb ?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak"trainer/global_step2:JPÏVDÛnkbj GDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak"trainer/global_step2:JP¹JäöWlbS 0DeviceStatsMonitor.on_train_batch_start/num_ooms"trainer/global_step2:JPòó§qmbm JDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current"trainer/global_step2:JPʆÆÞhnbd ADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed"trainer/global_step2:JP¿ axobt QDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated"trainer/global_step2:JP—ÄgKjpbf CDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed"trainer/global_step2:JP6g³ðqqbm JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated"trainer/global_step2:JPK4q]rbY 6DeviceStatsMonitor.on_train_batch_start/max_split_size"trainer/global_step2:JPqœ;nsbj GDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed"trainer/global_step2:JP~G…Zjtbf CDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated"trainer/global_step2:JP¤ l euba >DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed"trainer/global_step2:JP†|7lvbh EDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed"trainer/global_step2:JP]ÞZ#kwbg DDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated"trainer/global_step2:JPZv= gxbc @DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed"trainer/global_step2:JP{Íh±mybi FDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak"trainer/global_step2:JPܦµfzbb ?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak"trainer/global_step2:JPn9l{bh EDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak"trainer/global_step2:JPÌ@•Ír|bn KDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated"trainer/global_step2:JPÂ\g_}b[ 8DeviceStatsMonitor.on_train_batch_start/segment.all.peak"trainer/global_step2:JP¦*½e~ba >DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak"trainer/global_step2:JPDðònbj GDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current"trainer/global_step2:JP>Ñ—p€bk HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed"trainer/global_step2:JPR».Ìhbc @DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated"trainer/global_step2:JPìBeÚh‚bc @DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed"trainer/global_step2:JPxÈäoƒbj GDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed"trainer/global_step2:JPÏæP h„bc @DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed"trainer/global_step2:JPs^kÔd…b_ DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak"trainer/global_step2:JPÇfläo‡bj GDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current"trainer/global_step2:JP, «ªqˆbl IDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current"trainer/global_step2:JPÇØCh‰bc @DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed"trainer/global_step2:JP{vzZcŠb^ ;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak"trainer/global_step2:JP¥n«ut‹bo LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated"trainer/global_step2:JPìr9¤jŒbe BDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current"trainer/global_step2:JP¶Tobj GDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed"trainer/global_step2:JPh³5gŽbb ?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak"trainer/global_step2:JP[H7mbh EDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed"trainer/global_step2:JP5ÌœRqbl IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated"trainer/global_step2:JP-¿^d‘b_ DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 WEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚ 1338488646144 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚ 1326714355712 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 4552982528 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚41931264 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 PCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚58387268 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 5492441088 RGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚167302 UHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚75181568 NCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚179871 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚4260 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 122890752 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 4654587392 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 4628521086 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 QGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚41383 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚57299456 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 SADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚ 1331087822614 XHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 11707797270 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 5492441088 @7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚5235 J?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚163042 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 4517085184 KDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚59 IBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚76 ]MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 12468084736 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 4552982528 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17104896 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚54537728 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 123731968 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚321 _runtime‚ 15.57891688 VKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚111089 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4262 JBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚665 UKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚41457 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚321 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚321 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚262 RGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚179871 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚4260 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚ 1166281965568 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 5368709120 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚ 1326714355712 PCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚59036160 PIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚74 XFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚ 1178750050304 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 123731968 J?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚179869 M@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚59036160 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚ 1319380025344 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚262 F?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚59 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 122890752 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚41347908 SADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚ 1338429609984 aQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 12522622464 NCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚163042 \JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚ 1178825231872 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 RGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚111087 NCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚167302 P>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚ 1338429609984 UEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 11732359168 ODDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚152546 K@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚152470 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚202 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚422 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 843096576 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 5368709120 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚321 F>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17104896 XHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 11732359168 K@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚347173 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 GDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4847 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚41931264 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 D;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚5235 \LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 11749145178 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 5492441088 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚262 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚ 1326697250816 YIDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 11774290432 G/d—àN K@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚152470 NFDeviceStatsMon)n5'nitor.on_train_batch_start/inactive_split.small_pool.peak‚202 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚422 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 843096576 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 5368709120 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚321 F>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17104896 XHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 11732359168 K@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚347173 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 GDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4847 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚41931264 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 D;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚5235 \LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 11749145178 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 5492441088 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚262 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚ 1326697250816 YIDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 11774290432 GDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 WEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚ 1338488646144 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚ 1326714355712 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 4552982528 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚41931264 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 PCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚58387268 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 5492441088 RGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚167302 UHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚75181568 NCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚179871 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚4260 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 122890752 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 4654587392 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 4628521086 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 QGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚41383 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚57299456 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 SADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚ 1331087822614 XHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 11707797270 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 5492441088 @7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚5235 J?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚163042 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 4517085184 KDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚59 IBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚76 ]MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 12468084736 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 4552982528 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17104896 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚54537728 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 123731968 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚321 _runtime‚ 15.57891688 VKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚111089 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4262 JBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚665 UKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚41457 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚321 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚321 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚262 RGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚179871 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚4260 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚ 1166281965568 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 5368709120 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚ 1326714355712 PCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚59036160 PIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚74 XFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚ 1178750050304 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 123731968 J?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚179869 M@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚59036160 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚ 1319380025344 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚262 F?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚59 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 122890752 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚41347908 SADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚ 1338429609984 aQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 12522622464 NCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚163042 \JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚ 1178825231872 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 RGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚111087 NCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚167302 P>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚ 1338429609984 UEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 11732359168 ODDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚152546?¹Éøf˜ba >DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak"trainer/global_step2:JPx×&j™be BDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak"trainer/global_step2:JPfk’]šbX 5DeviceStatsMonitor.on_train_batch_end/num_device_free"trainer/global_step2:JPx¥{ej›be BDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated"trainer/global_step2:JPŒ°Ù[iœbd ADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed"trainer/global_step2:JP®ÜÙ¨lbg DDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak"trainer/global_step2:JP’–Snžbi FDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed"trainer/global_step2:JPßÊcreŸb` =DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed"trainer/global_step2:JPf[†òs bn KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed"trainer/global_step2:JP›t6†w¡br ODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated"trainer/global_step2:JP¯ój_h¢bc @DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed"trainer/global_step2:JPÎ< hg£bb ?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current"trainer/global_step2:JPÿÜ…bg¤bb ?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed"trainer/global_step2:JP£æ„Af¥ba >DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak"trainer/global_step2:JP›Ò†ù^¦bY 6DeviceStatsMonitor.on_train_batch_end/segment.all.peak"trainer/global_step2:JP>fÑ…b§b] :DeviceStatsMonitor.on_train_batch_end/active.all.allocated"trainer/global_step2:JP*:ƒ-p¨bk HDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current"trainer/global_step2:JP/ Ýk©bf CDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current"trainer/global_step2:JP’ñ¸mªbh EDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed"trainer/global_step2:JP¼,üjo«bj GDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current"trainer/global_step2:JPÿ¿m¬bh EDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed"trainer/global_step2:JPæcæf­ba >DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed"trainer/global_step2:JPn£Dzf®ba >DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current"trainer/global_step2:JP£þ®¾k¯bf CDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed"trainer/global_step2:JPÕµvøm°bh EDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current"trainer/global_step2:JPå–j±be BDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current"trainer/global_step2:JPa -Òm²bh EDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak"trainer/global_step2:JP ØB|i³bd ADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated"trainer/global_step2:JP{šCZj´be BDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak"trainer/global_step2:JPµ¦Î8bµb] :DeviceStatsMonitor.on_train_batch_end/allocation.all.freed"trainer/global_step2:JPfèKdh¶bc @DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak"trainer/global_step2:JPz¤¥õm·bh EDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated"trainer/global_step2:JPì"žp¸bk HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated"trainer/global_step2:JP™ˆî7_¹bZ 7DeviceStatsMonitor.on_train_batch_end/segment.all.freed"trainer/global_step2:JPLaõlºbg DDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak"trainer/global_step2:JPêüÉd»b_ DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed"trainer/global_step2:JPH¨ÝhÒbc @DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current"trainer/global_step2:JP¸2m,mÓbh EDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated"trainer/global_step2:JPØÛjéqÔbl IDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated"trainer/global_step2:JPGO£mÕbh EDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed"trainer/global_step2:JPñÈ¥ijÖbe BDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated"trainer/global_step2:JP'?Š=j×be BDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated"trainer/global_step2:JP•ÍÒkØbf CDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed"trainer/global_step2:JP#aÂhÙbc @DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current"trainer/global_step2:JP:ÅêíqÚbl IDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated"trainer/global_step2:JP w`Ûb[ 8DeviceStatsMonitor.on_train_batch_end/active.all.current"trainer/global_step2:JPÚ"?pÜbk HDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current"trainer/global_step2:JPr±ÍùnÝbi FDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed"trainer/global_step2:JPkxðUfÞba >DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated"trainer/global_step2:JP±×èeßb` =DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak"trainer/global_step2:JP<È”L^àbY 6DeviceStatsMonitor.on_train_batch_end/num_device_alloc"trainer/global_step2:JPy±«Ügább ?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak"trainer/global_step2:JPÒkaiâbd ADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current"trainer/global_step2:JPéüݾfãba >DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed"trainer/global_step2:JPruù­uäbp MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current"trainer/global_step2:JPÍ,'Ê`åb[ 8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections"trainer/global_step2:JPì;ãúlæbg DDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated"trainer/global_step2:JPÊ:¢cçb^ ;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated"trainer/global_step2:JP[8’dèb_ DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed"trainer/global_step2:JP÷–f gˆbb ?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current"trainer/global_step2:JPï=¹i‰bd ADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed"trainer/global_step2:JPE©$&rŠbm JDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak"trainer/global_step2:JP™x©r‹bm JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated"trainer/global_step2:JPˆi h]ŒbX 5DeviceStatsMonitor.on_train_batch_end/active.all.peak"trainer/global_step2:JPNk¨+mbh EDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current"trainer/global_step2:JPÙA°û\ŽbW 4DeviceStatsMonitor.on_train_batch_end/max_split_size"trainer/global_step2:JPÜ~;lbg DDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak"trainer/global_step2:JP™‰XÉhbc @DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated"trainer/global_step2:JPN‚k‘bf CDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak"trainer/global_step2:JPr2£l’bg DDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak"trainer/global_step2:JPc¾ÿQ.“) _wandbruntime‚15  _runtime‚15•ë4§"”çL C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚ 1345582858240 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 G@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚59 PEDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚183424 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 123731968 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 IBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚59 MBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚155521 SCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 11963427840 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 5492441088 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 5368709120 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚4268 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17104896 VFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 11963427840 I>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚354019 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚422 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚321 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 NADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚58387316 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚20643840 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚321 % _timestamp‚1.7866586672157938e+09 EDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚155458 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 LADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚183422 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 831258624 ZJDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 11979717838 >5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5237 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17104896 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 5368709120 R@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚ 1365042055680 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 843096576 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 123731968 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654588416 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 122891776 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 _step‚2 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 5492441088 LADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚166329 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚223 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚ 1345565818880 H=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚183422 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚ 1189329338368 _ODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 12766077440 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 Q?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚ 1357504188762 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521094 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚321 E:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚354019 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚41934336 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 OEDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚42166 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 5368709120 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 K>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59039232 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚ 1353019588608 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚41934336 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 4517085184 LADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚183424 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 4552982528 E:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚349751 H@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚665 PEDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚170595 ZHDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚ 1202116059648 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚202 B—"q&•ëL NGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚61 [KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 12713639936 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚262 LADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚170595 TIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚113294 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 _runtime‚ 15.746225627 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚321 B9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚5237 DDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚155458 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 LADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚183422 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 831258624 ZJDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 11979717838 >5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5237 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17104896 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 5368709120 R@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚ 1365042055680 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 843096576 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 123731968 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654588416 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 122891776 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 _step‚2 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 5492441088 LADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚166329 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚223 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚ 1345565818880 H=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚183422 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚ 1189329338368 _ODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 12766077440 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 Q?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚ 1357504188762 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521094 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚321 E:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚354019 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚41934336 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 OEDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚42166 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 5368709120 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 K>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59039232 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚ 1353019588608 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚41934336 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 4517085184 LADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚183424 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 4552982528 E:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚349751 H@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚665 PEDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚170595 ZHDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚ 1202116059648 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚202 BDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 G@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚59 PEDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚183424 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 123731968 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 IBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚59 MBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚155521 SCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 11963427840 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 5492441088 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 5368709120 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚4268 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17104896 VFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 11963427840 I>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚354019 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚422 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚321 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 NADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚58387316 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚20643840 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚321 % _timestamp‚1.7866586672157938e+09 EDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚ 2666133265408 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚ 2642831245312 JBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚665 % _timestamp‚1.7866586796656902e+09 C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚4262 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚223 SADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚ 2651939310022 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 5368709120 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚262 NCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚328144 IBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚63 BDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚ 2628669702144 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚41347908 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17104896 PCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚59036160 aQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 24768456704 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚321 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 5368709120 J?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚328144 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚422 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚ 2318650605568 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 YIDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 23361056256 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 4517085184 F?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚59 TBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚ 2666192301568 GDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 RGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚357521 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 123731968 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚321 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚41931264 SADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚ 2666133265408 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4262 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚4260 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 4552982528 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚41931264 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚202 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 831258624 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚321 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚262 IBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚59 G>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4849 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚ 2642814140416 KBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚4849 \LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 23361056256 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚ 2318671249408 J?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚357519 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚20643840 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚48246272 ]MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 24720210432 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 122306414 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 C8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚685663 K@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚689925 RGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚332404 \JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚ 2343439706112 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 4628521094 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 5492441088 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 122891776 PIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚61 XHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 23319124992 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 123731968¸r.Çg'¤áN SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 4552982528 VKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚221339 UHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚68890112 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 RGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚357521 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 123731968 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚321 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚41931264 SADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚ 2666133265408 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4262 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚4260 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 4552982528 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚41931264 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚202 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 831258624 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚321 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚262 IBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚59 G>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4849 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚ 2642814140416 KBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚4849 \LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 23361056256 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚ 2318671249408 J?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚357519 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚20643840 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚48246272 ]MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 24720210432 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 122306414 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 C8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚685663 K@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚689925 RGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚332404 \JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚ 2343439706112 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 4628521094 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 5492441088 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 122891776 PIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚61 XHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 23319124992 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 123731968 _runtime‚ 28.198873418 GDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚ 2666133265408 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚ 2642831245312 JBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚665 % _timestamp‚1.7866586796656902e+09 C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚4262 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚223 SADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚ 2651939310022 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 5368709120 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚262 NCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚328144 IBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚63 BDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚ 2628669702144 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚41347908 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17104896 PCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚59036160 aQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 24768456704 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚321 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 5368709120 J?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚328144 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚422 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚ 2318650605568 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 YIDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 23361056256 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 4517085184 F?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚59 TBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚ 2666192301568 GDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59039232 E:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚696771 E8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 NADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚58387316 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚ 2654872535040 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 122306414 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚262 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚ 2669153583104 WGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 23592128000 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 122891776 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 122891776 B9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚5237 H@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚665 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 831258624 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 5368709120 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚4266 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚ 2669153583104 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚202 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 5368709120 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17104896 _step‚6 SIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚83820 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 4552982528 I@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚4849 TIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚223544 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚ 2341697978368 R@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚ 2692745711104 H=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚361072 I>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚696771 PEDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚335697 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚20643840 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚262 SFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚70984192 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚321 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚59395584 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654588416 Q?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚ 2678355676170 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚ 2654855495680 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 5492441088 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 5492441088 NADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚59039232 ZJDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 23592128000 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 D=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚59 _runtime‚ 28.355590179 I>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚307289 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚ 2341718622208 IBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚59 LADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚335697 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17104896 OEDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚83747 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 123731968 VFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 23550193664 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚50340352 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 UCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚ 2678414063486 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚223 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 123731968 N6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚321 DDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 5368709120 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚321 PEDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚223542 VDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚ 2366660401664 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 123731968 Q?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚ 2692686671872 H=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚331431 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚ 2669136478208 MBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚307364 [KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 24962423296 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 >5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5237 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚4266 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚321 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521094 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 A6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚692503®Îq&§ëL A6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚692503 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 K>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59039232 E:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚696771 E8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 NADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚58387316 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚ 2654872535040 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 122306414 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚262 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚ 2669153583104 WGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 23592128000 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 122891776 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 122891776 B9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚5237 H@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚665 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 831258624 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 5368709120 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚4266 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚ 2669153583104 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚202 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 5368709120 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17104896 _step‚6 SIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚83820 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 4552982528 I@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚4849 TIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚223544 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚ 2341697978368 R@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚ 2692745711104 H=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚361072 I>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚696771 PEDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚335697 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚20643840 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚262 SFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚70984192 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚321 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚59395584 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654588416 Q?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚ 2678355676170 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚ 2654855495680 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 5492441088 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 5492441088 NADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚59039232 ZJDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 23592128000 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 D=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚59 _runtime‚ 28.355590179 I>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚307289 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚ 2341718622208 IBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚59 LADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚335697 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17104896 OEDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚83747 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 123731968 VFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 23550193664 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚50340352 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 UCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚ 2678414063486 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚223 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 123731968 N6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚321 DDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 5368709120 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚321 PEDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚223542 VDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚ 2366660401664 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 123731968 Q?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚ 2692686671872 H=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚331431 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚ 2669136478208 MBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚307364 [KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 24962423296 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 >5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5237 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚4266 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚321 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521094 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0¿æ=.¨) _wandbruntime‚28  _runtime‚28  k©å 0loss/FRt1/detection_loss‚0.49844643724365206 /loss/FRt2/detection_loss‚0.5482431155624354 %loss/total_loss‚5.552856352403351 % _timestamp‚1.7866586798263435e+09 _runtime‚ 28.356194722 _step‚7 /loss/FRt1/assignment_loss‚2.043483248855069 0loss/FRt2/assignment_loss‚2.4626835507421947 epoch‚0 trainer/global_step‚99ñÀ°Óoªé _runtime‚ 28.356194722 _step‚7 /loss/FRt1/assignment_loss‚2.043483248855069 0loss/FRt2/assignment_loss‚2.4626835507421947 epoch‚0 trainer/global_step‚99 0loss/FRt1/detection_loss‚0.49844643724365206 /loss/FRt2/detection_loss‚0.5482431155624354 %loss/total_loss‚5.552856352403351 % _timestamp‚1.7866586798263435e+09B¹xà”«‚(:‰ ùþøÓ‚¾±á gpu.0.gpu‚0.0 gpu.0.memory‚0, gpu.0.memoryAllocated‚7.0288848876953125) gpu.0.memoryAllocatedBytes‚ 6037766144 gpu.0.temp‚33.0 gpu.0.powerWatts‚180.854' gpu.0.enforcedPowerLimitWatts‚500.0 gpu.0.powerPercent‚36.1708 gpu.0.smClock‚1410 gpu.0.memoryClock‚1593$ gpu.0.correctedMemoryErrors‚1955# gpu.0.uncorrectedMemoryErrors‚09‡q¬‚(:æ ùþøÓÜ•¼ä disk.drbd2165.out‚0 disk.nvme0c0n1.in‚ 7.5546875) proc.memory.availableMB‚ 6105.09765625 proc.cpu.threads‚17 network.recv‚124829 disk.sda1.in‚0 disk.sda1.out‚0 disk.dm-0.in‚ 7.5546875 disk.sda3.in‚ 161.1640625 disk.sda3.out‚ 0.69140625% disk.nvme0c0n1.out‚52.10205078125 disk.sda.in‚ 161.1640625* disk./.usagePercent‚30.603987516007642# disk.nvme0n1.out‚52.10205078125 disk.dm-1.in‚ 161.1640625% memory_percent‚62.737441062927246! proc.memory.rssMB‚ 2251.484375) proc.memory.percent‚13.74197006225586 cpu‚20.162797296013668$ disk./.usageGB‚456.0215301513672 disk.sda2.out‚0 disk.dm-1.out‚ 0.69140625 disk.dm-0.out‚52.10205078125 disk.sda.out‚ 0.69140625 network.sent‚126326 disk.sda2.in‚0 disk.nvme0n1.in‚ 7.5546875 disk.drbd2165.in‚0Üâ7.­) _wandbruntime‚41  _runtime‚41b¹c ¬®¦ _step‚8  lr-AdamW/pg1‚ 0.00055875  lr-AdamW/pg2‚ 0.00055875 trainer/global_step‚149 % _timestamp‚1.7866586930722206e+09 _runtime‚ 41.602304555]ðS°¯ª  lr-AdamW/pg1‚ 0.00055875  lr-AdamW/pg2‚ 0.00055875 trainer/global_step‚149 % _timestamp‚1.7866586930722206e+09 _runtime‚ 41.602304555 _step‚8Šì•R.°) _wandbruntime‚41  _runtime‚41w· 'j'±äN G>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4849 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 122891776 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚ 3958948134912 XHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 34905890816 NCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚497506 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17104896 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚321 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 XHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 34831418486 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 L@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚1032677 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚223 PCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚58387268 F>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚ 3958931030016 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚59395584 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚ 3937976418304 UEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 34905890816 PCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚59036160 NCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚535169 ]MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 36979932160 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚41347908 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 123731968 @7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚5237 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 4552982528 \LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 34872766394 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚321 TBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚ 3993895956992 RGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚124544 ODDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚456200 F?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚59 aQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 37030275584 _runtime‚ 41.606956697 _step‚9 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚202 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚ 3471019245568 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚321 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚321 C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚4262 P>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚ 3993836920832 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 122306414 J?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚493246 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚4260 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 831258624 HDeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4262 JBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚665 IBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚69 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 4654588416 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚ 3958948134912 KDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚59 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 XFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚ 3507999177728 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚41931264 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 122891776 RGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚535171 WEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚ 3972849184698 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚ 3937959378944 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 trainer/global_step‚149 J?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚535169 RGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚497506 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 5368709120 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 5368709120 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚262 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚ 3958931030016 NCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚493246 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚20643840 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 BDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4849 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 122891776 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚ 3958948134912 XHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 34905890816 NCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚497506 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17104896 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚321 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 XHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 34831418486 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 L@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚1032677 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚223 PCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚58387268 F>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚ 3958931030016 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚59395584 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚ 3937976418304 UEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 34905890816 PCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚59036160 NCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚535169 ]MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 36979932160 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚41347908 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 123731968 @7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚5237 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 4552982528 \LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 34872766394 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚321 TBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚ 3993895956992 RGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚124544 ODDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚456200 F?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚59 aQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 37030275584 _runtime‚ 41.606956697 _step‚9 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚202 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚ 3471019245568 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚321 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚321 C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚4262 P>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚ 3993836920832 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 122306414 J?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚493246 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚4260 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 831258624 HDeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4262 JBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚665 IBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚69 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 4654588416 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚ 3958948134912 KDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚59 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 XFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚ 3507999177728 ’^m.³) _wandbruntime‚41  _runtime‚41ù>lŒu&´ïL L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 5492441088 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚59395584 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚321 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 WGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 35178893824 LADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚538722 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 123731968 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚41934336 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 5492441088 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚321 G@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚82 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 831258624 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 5368709120 IBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚59 F:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚1039523 I>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚459131 ZHDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚ 3531363253760 >5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5237 NADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚59039232 E6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚321 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 4654588416 Q?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚ 4020390327296 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 122891776 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 4552982528 VFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 35136959488 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 UCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚ 3999265550894 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 4517085184 ZJDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 35103339054 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚321 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚262 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521094 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 VFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 35061991098 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 5492441088 G@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚59 _step‚10 NDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59039232 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 VDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚ 3531292269568 [KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 37225651200 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚41347956 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚262 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚ 3985270472704 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 122891776 PEDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚538724 LADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚496533 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚262 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚ 3985253367808 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 4552982528 ZJDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 35178893824 SFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚70984192 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚ 3494087262208 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 LADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚500799 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17104896 PEDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚500799 PEDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚333792 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚ 3964162211840 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚4268 DDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654588416 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚422 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚4266 J>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚1039523 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚4266 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚202 Q?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚ 3999207163578 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 trainer/global_step‚149 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚ 3985253367808 NADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚58387316 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚41934336 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚50340352 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚ 3964145172480üÖâ‘y&µóL UCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚ 3985253367808 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 4552982528 ZJDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 35178893824 SFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚70984192 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚ 3494087262208 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 LADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚500799 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17104896 PEDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚500799 PEDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚333792 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚ 3964162211840 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚4268 DDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654588416 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚422 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚4266 J>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚1039523 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚4266 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚202 Q?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚ 3999207163578 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 trainer/global_step‚149 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚ 3985253367808 NADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚58387316 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚41934336 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚50340352 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚ 3964145172480 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 5492441088 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚59395584 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚321 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 WGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 35178893824 LADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚538722 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 123731968 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚41934336 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 5492441088 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚321 G@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚82 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 831258624 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 5368709120 IBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚59 F:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚1039523 I>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚459131 ZHDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚ 3531363253760 >5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5237 NADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚59039232 E6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚321 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 4654588416 Q?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚ 4020390327296 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 122891776 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 4552982528 VFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 35136959488 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 UCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚ 3999265550894 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 4517085184 ZJDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 35103339054 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚321 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚262 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521094 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 VFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 35061991098 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 5492441088 G@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚59 _step‚10 NDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59039232 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 VDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚ 3531292269568 [KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 37225651200 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚41347956 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚262 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚ 3985270472704 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 122891776 PEDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚538724 LADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚496533 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚262  -.¶) _wandbruntime‚41  _runtime‚41'Ñ‹l·æ /loss/FRt2/detection_loss‚0.3608613875081308 epoch‚0 % _timestamp‚1.7866586932580082e+09 _runtime‚ 41.787870167 _step‚11 /loss/FRt2/assignment_loss‚2.148008164553748 %loss/total_loss‚5.773228122016324 trainer/global_step‚149 0loss/FRt1/assignment_loss‚2.5608368243140984 /loss/FRt1/detection_loss‚0.7035217456403472(ÛRyp¸ê /loss/FRt2/assignment_loss‚2.148008164553748 %loss/total_loss‚5.773228122016324 trainer/global_step‚149 0loss/FRt1/assignment_loss‚2.5608368243140984 /loss/FRt1/detection_loss‚0.7035217456403472 /loss/FRt2/detection_loss‚0.3608613875081308 epoch‚0 % _timestamp‚1.7866586932580082e+09 _runtime‚ 41.787870167 _step‚11 òû¼–¹‚(:‹ ˆÿøÓÇÛá gpu.0.gpu‚64.0 gpu.0.memory‚12, gpu.0.memoryAllocated‚7.0288848876953125) gpu.0.memoryAllocatedBytes‚ 6037766144 gpu.0.temp‚35.0 gpu.0.powerWatts‚122.684' gpu.0.enforcedPowerLimitWatts‚500.0 gpu.0.powerPercent‚24.5368 gpu.0.smClock‚1410 gpu.0.memoryClock‚1593$ gpu.0.correctedMemoryErrors‚1955# gpu.0.uncorrectedMemoryErrors‚0ÉÞ…[[º‚(:Ð ˆÿøÓÃÊÿâ disk.dm-0.out‚97.26220703125 disk.sda1.in‚0 disk.sda2.out‚0# disk.nvme0n1.out‚97.26220703125 cpu‚21.446591805606545 disk.sda1.out‚0) disk./.usagePercent‚30.60399519623367 disk.sda3.in‚ 161.1640625 disk.sda3.out‚ 0.82421875 disk.dm-1.in‚ 161.1640625 disk.dm-1.out‚ 0.82421875 disk.drbd2165.out‚0( proc.memory.availableMB‚ 6103.7890625 network.sent‚220481 disk.sda.in‚ 161.1640625 disk.nvme0n1.in‚21 disk.drbd2165.in‚0" proc.memory.rssMB‚ 2251.4921875) proc.memory.percent‚13.74201774597168 proc.cpu.threads‚17 disk.sda.out‚ 0.82421875 network.recv‚157824% disk./.usageGB‚456.02164459228516 disk.dm-0.in‚21 disk.nvme0c0n1.in‚21% disk.nvme0c0n1.out‚97.26220703125 disk.sda2.in‚0$ memory_percent‚62.74542808532715ÉçÑ+.») _wandbruntime‚54  _runtime‚54¡`ö¦Ã¼½ & lr-AdamW/pg1‚0.0007462500000000001 & lr-AdamW/pg2‚0.0007462500000000001 trainer/global_step‚199 % _timestamp‚1.7866587061324468e+09 _runtime‚ 54.662528898 _step‚12d¼³ÔǽÁ & lr-AdamW/pg1‚0.0007462500000000001 & lr-AdamW/pg2‚0.0007462500000000001 trainer/global_step‚199 % _timestamp‚1.7866587061324468e+09 _runtime‚ 54.662528898 _step‚12 vdŒk.¾) _wandbruntime‚54  _runtime‚54ñ²¯ó¿åN \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚20643840 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 122306414 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 PCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚58387268 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 5492441088 PCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚59036160 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚223 UHDeviceStatsMonitor.on_train_batch_stž6¶íx$art/inactive_split_bytes.all.current‚60501504 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚ 4623408529408 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚ 5247266095104 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚262 NCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚662608 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚ 5275065024512 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 123731968 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 % _timestamp‚1.7866587061334484e+09 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4262 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 123731968 PIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚65 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 _runtime‚ 54.666917602 J?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚712819 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 IBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚59 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 4654588416 aQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 49319021568 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚39857664 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚ 5247249055744 \LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 46434577002 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 5492441088 D8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚1371167 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚41931264 @7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚5237 SADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚ 5321540576256 HDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚ 5321540576256 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 4552982528 \LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 46534587904 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚ 4623387885568 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 123731968 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚ 5275065024512 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 4552982528 YIDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 46534587904 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 WEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚ 5321599612416 HDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 RGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚712821 JBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚665 ]MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 49279163904 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚ 5275047919616 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚4260 NCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚658348 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚202 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 122891776 L@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚1375429 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 5492441088 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚262 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 5368709120 KDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚59 VKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚441839 \JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚ 4672727550976 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚321 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 5368709120 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚4260 G>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4849 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚ 5275047919616 UEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 46492656640 NCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚712819 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚59395584 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 D;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚5237 VKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚166537‡¾ôÐo'ÀéN PCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚58387268 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 5492441088 PCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚59036160 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚223 UHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚60501504 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚ 4623408529408 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚ 5247266095104 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚262 NCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚662608 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚ 5275065024512 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 123731968 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 % _timestamp‚1.7866587061334484e+09 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4262 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 123731968 PIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚65 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 _runtime‚ 54.666917602 J?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚712819 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 IBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚59 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 4654588416 aQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 49319021568 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚39857664 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚ 5247249055744 \LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 46434577002 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 5492441088 D8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚1371167 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚41931264 @7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚5237 SADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚ 5321540576256 HDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚ 5321540576256 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 4552982528 \LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 46534587904 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚ 4623387885568 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 123731968 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚ 5275065024512 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 4552982528 YIDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 46534587904 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 WEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚ 5321599612416 HDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 RGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚712821 JBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚665 ]MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 49279163904 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚ 5275047919616 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚4260 NCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚658348 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚202 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 122891776 L@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚1375429 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 5492441088 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚262 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 5368709120 KDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚59 VKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚441839 \JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚ 4672727550976 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚321 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 5368709120 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚4260 G>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4849 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚ 5275047919616 UEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 46492656640 NCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚712819 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚59395584 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 D;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚5237 VKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚166537 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚20643840 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 122306414 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 ]ˆ .Á) _wandbruntime‚54  _runtime‚54`_Du&ÂïL B9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚5237 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 843096576 [KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 49517706752 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚59395584 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 LADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚661635 VDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚ 4695952965120 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 5492441088 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚262 H=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚661635 WGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 46765659648 J>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚1382275 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚4266 TIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚444044 PEDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚444042 UCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚ 5320117038302 NADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚58387316 Q?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚ 5348093982720 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17104896 PEDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚665901 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚ 4646455902208 VFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 46623801706 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 % _timestamp‚1.7866587063000886e+09 R@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚ 5348153021952 NGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚82 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 123731968 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚20643840 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚ 5301387362304 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17104896 ZJDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 46765659648 VFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 46723725312 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521094 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚41347956 MBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚611443 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚321 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 F:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚1382275 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚4268 LADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚665901 I@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚4849 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚223 PEDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚167317 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚ 4646435258368 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 5368709120 H=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚716372 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚ 5301387362304 PEDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚716374 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚262 >5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5237 LADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚716374 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 4552982528 F:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚1378007 G@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚59 H@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚665 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 4517085184 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 K>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59039232 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 4552982528 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 Q?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚ 5320058650986 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 5368709120 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 4654588416 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 NADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚59039232 IBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚59 trainer/global_step‚199 _step‚14 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654588416 EDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 E6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚321 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 123731968 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚321 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚202 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚ 5273434849280 SCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 46723725312 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 122891776 LADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚716372 _ODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 49565949952 TIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚167399 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 I>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚611359 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚48243200 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚ 5273451888640 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 122306414 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚422 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚321 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚ 5301370257408 B8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 Q?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚ 5320058650986 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 5368709120 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 4654588416 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 NADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚59039232 IBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚59 trainer/global_step‚199 _step‚14 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654588416 EDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 E6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚321 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 123731968 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚321 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚202 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚ 5273434849280 SCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 46723725312 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 122891776 LADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚716372 _ODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 49565949952 TIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚167399 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 I>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚611359 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚48243200 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚ 5273451888640 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 122306414 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚422 ;5DeviceStlˆ'¤&atsMonitor.on_train_batch_end/num_device_free‚0 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚321 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚ 5301370257408 BDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚1382275 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚4266 TIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚444044 PEDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚444042 UCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚ 5320117038302 NADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚58387316 Q?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚ 5348093982720 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17104896 PEDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚665901 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚ 4646455902208 VFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 46623801706 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 % _timestamp‚1.7866587063000886e+09 R@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚ 5348153021952 NGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚82 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 123731968 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚20643840 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚ 5301387362304 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17104896 ZJDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 46765659648 VFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 46723725312 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521094 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚41347956 MBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚611443 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚321 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 F:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚1382275 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚4268 LADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚665901 I@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚4849 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚223 PEDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚167317 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚ 4646435258368 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 5368709120 H=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚716372 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚ 5301387362304 PEDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚716374 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚262 >5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5237 LADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚716374 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 4552982528 F:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚1378007 G@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚59 H@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚665 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 4517085184 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 K>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59039232 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 4552982528 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360â PF.Ä) _wandbruntime‚54  _runtime‚54n"y€lÅæ /loss/FRt1/detection_loss‚0.5112876062828344 /loss/FRt2/detection_loss‚0.8608584243077881 %loss/total_loss‚5.482863033247499 epoch‚0 /loss/FRt1/assignment_loss‚1.490679290414109 trainer/global_step‚199 % _timestamp‚1.7866587063011317e+09 _runtime‚ 54.831001708 _step‚15 0loss/FRt2/assignment_loss‚2.6200377122427674¿îÏãpÆê /loss/FRt1/detection_loss‚0.5112876062828344 /loss/FRt2/detection_loss‚0.8608584243077881 %loss/total_loss‚5.482863033247499 epoch‚0 /loss/FRt1/assignment_loss‚1.490679290414109 trainer/global_step‚199 % _timestamp‚1.7866587063011317e+09 _runtime‚ 54.831001708 _step‚15 0loss/FRt2/assignment_loss‚2.6200377122427674ê‹–Ç‚(:‹ —ÿøÓáâ£á gpu.0.gpu‚64.0 gpu.0.memory‚12, gpu.0.memoryAllocated‚7.0288848876953125) gpu.0.memoryAllocatedBytes‚ 6037766144 gpu.0.temp‚35.0 gpu.0.powerWatts‚162.323' gpu.0.enforcedPowerLimitWatts‚500.0 gpu.0.powerPercent‚32.4646 gpu.0.smClock‚1410 gpu.0.memoryClock‚1593$ gpu.0.correctedMemoryErrors‚1955# gpu.0.uncorrectedMemoryErrors‚0>»]È‚(:Ò —ÿøÓ©Ôä network.sent‚289293 disk.sda3.in‚ 161.1640625 disk.nvme0c0n1.in‚21 disk.sda.out‚ 1.23828125 disk.sda1.in‚0$ disk.nvme0n1.out‚109.06591796875 disk.drbd2165.in‚0* disk./.usagePercent‚30.603995708248732$ disk./.usageGB‚456.0216522216797& disk.nvme0c0n1.out‚109.06591796875 disk.sda2.out‚0 disk.nvme0n1.in‚21 disk.dm-0.in‚21$ memory_percent‚62.75043487548828& proc.memory.availableMB‚ 6102.96875 network.recv‚185430 disk.sda3.out‚ 1.23828125 disk.sda.in‚ 161.1640625 disk.dm-1.in‚ 161.1640625 disk.dm-1.out‚ 1.23828125 disk.drbd2165.out‚0! disk.dm-0.out‚109.06591796875# proc.memory.rssMB‚ 2251.49609375 disk.sda1.out‚0) proc.memory.percent‚13.74204158782959 cpu‚22.314460760111885 proc.cpu.threads‚17 disk.sda2.in‚0H€á.É) _wandbruntime‚67  _runtime‚67`‡êPÃʽ _runtime‚ 67.189844298 _step‚16 & lr-AdamW/pg1‚0.0009337500000000001 & lr-AdamW/pg2‚0.0009337500000000001 trainer/global_step‚249 % _timestamp‚1.7866587186598046e+09‰PrÇËÁ & lr-AdamW/pg1‚0.0009337500000000001 & lr-AdamW/pg2‚0.0009337500000000001 trainer/global_step‚249 % _timestamp‚1.7866587186598046e+09 _runtime‚ 67.189844298 _step‚16÷B¡.Ì) _wandbruntime‚67  _runtime‚67#¯h'ÍâN XHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 57955039702 IBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚59 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 843096576 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 831258624 aQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 61599239168 KDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚59 C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚4262 L@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚1718181 \LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 57996387610 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 123731968 PCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚59036160 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚321 P>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚ 6649244231680 NCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚890469 ODDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚760366 \JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚ 5837376408576 @7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚5237 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 4552982528 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚20643840 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚59395584 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 122306414 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚321 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 NCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚827710 UEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 58079422464 D;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚5237 WEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚ 6614552159514 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17104896 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚41931264 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚ 5775777169408 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 HDeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4262 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚262 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚321 NCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚890471 M@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚59036160 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚ 6591181914112 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚ 6591164809216 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚41931264 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 122891776 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 \LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 58121353728 PCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚58387268 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 5368709120 trainer/global_step‚249 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚ 6591181914112 HDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4849 YIDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 58121353728 RGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚890471 UHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚68890112 J?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚890469 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 4628521094 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚223 SADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚ 6649244231680 XHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 58079422464 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 122891776 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 4517085184 NCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚823450 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚262 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚262 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 D8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚1713919 TBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚ 6649303267840 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚ 6591164809216 IBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚77 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚ 6556555771904 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 F?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚59 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚4260 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 4654588416 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚48246272 F>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 VKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚208277 RGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚208202 ]MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 61550992896 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 123731968 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 _runtime‚ 67.193914 VKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚552089 RGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚552087 WEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚ 6649303267840 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚429 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚ 5775756525568 SADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚ 6614493772246 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚321 PIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚75ƒœº@l'ÎæN PCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚59036160 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚321 P>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚ 6649244231680 NCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚890469 ODDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚760366 \JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚ 5837376408576 @7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚5237 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 4552982528 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚20643840 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚59395584 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 122306414 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚321 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 NCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚827710 UEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 58079422464 D;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚5237 WEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚ 6614552159514 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17104896 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚41931264 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚ 5775777169408 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 HDeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4262 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚262 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚321 NCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚890471 M@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚59036160 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚ 6591181914112 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚ 6591164809216 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚41931264 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 122891776 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 \LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 58121353728 PCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚58387268 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 5368709120 trainer/global_step‚249 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚ 6591181914112 HDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4849 YIDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 58121353728 RGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚890471 UHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚68890112 J?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚890469 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 4628521094 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚223 SADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚ 6649244231680 XHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 58079422464 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 122891776 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 4517085184 NCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚823450 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚262 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚262 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 D8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚1713919 TBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚ 6649303267840 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚ 6591164809216 IBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚77 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚ 6556555771904 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 F?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚59 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚4260 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 4654588416 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚48246272 F>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 VKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚208277 RGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚208202 ]MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 61550992896 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 123731968 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 _runtime‚ 67.193914 VKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚552089 RGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚552087 WEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚ 6649303267840 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚429 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚ 5775756525568 SADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚ 6614493772246 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚321 PIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚75 XHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 57955039702 IBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚59 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 843096576 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 831258624 aQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 61599239168 KDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚59 C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚4262 L@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚1718181 \LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 57996387610 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 123731968b}‰ž.Ï) _wandbruntime‚67  _runtime‚67 åG“ ÐïL WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚59395584 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚ 6582724526080 VFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 58185612314 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 4654588416 UCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚ 6675856677376 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚4266 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 Q?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚ 6640910138394 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 5368709120 LADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚826737 E6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚321 LADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚894022 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 123731968 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚48243200 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚262 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚ 5798824542208 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 831258624 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 122306414 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚4268 H=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚894022 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚ 6617487147008 VFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 58310491136 VDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚ 5860597988864 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521094 F:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚1725027 TIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚209109 PEDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚209045 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚41347956 % _timestamp‚1.7866587188344953e+09 ;5DeviceStatsMonitor.on_train_batch_end/num_dev¬ÒÞâice_free‚0 J>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚1725027 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 DDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654588416 PEDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚894024 MBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚763403 G@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚66 TIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚554294 _ODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 61842333696 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚262 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 122891776 H@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚665 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚429 trainer/global_step‚249 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 4517085184 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 5368709120 R@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚ 6675856677376 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚ 6582741565440 D=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚59 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚223 B6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚1720759 NDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚763337 PEDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚554292 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚41934336 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚41934336 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17104896 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 UCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚ 6640968525710 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 G@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚59 SFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚68887040 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚ 5798803898368 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 5492441088 _step‚18 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 4552982528 E6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚321 NADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚58387316 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 123731968 LADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚894024 K>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59039232 ZHDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚ 5860666875904 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17104896 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 123731968 B9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚5237 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 5368709120 H=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚826737 SCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 58310491136 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 5492441088 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚321 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 LADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚831003 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚20643840 [KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 61794090496 >5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5237 PEDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚831003 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 IBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚59 ZJDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 58352425472 NGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚64&²pþy&ÑóL >5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5237 PEDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚831003 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 IBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚59 ZJDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 58352425472 NGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚64 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚59395584 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚ 6582724526080 VFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 58185612314 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 4654588416 UCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚ 6675856677376 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚4266 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 Q?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚ 6640910138394 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 5368709120 LADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚826737 E6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚321 LADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚894022 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 123731968 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚48243200 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚262 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚ 5798824542208 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 831258624 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 122306414 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚4268 H=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚894022 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚ 6617487147008 VFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 58310491136 VDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚ 5860597988864 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521094 F:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚1725027 TIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚209109 PEDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚209045 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚41347956 % _timestamp‚1.7866587188344953e+09 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 J>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚1725027 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 DDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654588416 PEDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚894024 MBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚763403 G@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚66 TIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚554294 _ODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 61842333696 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚262 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 122891776 H@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚665 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚429 trainer/global_step‚249 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 4517085184 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 5368709120 R@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚ 6675856677376 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚ 6582741565440 D=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚59 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚223 B6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚1720759 NDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚763337 PEDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚554292 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚41934336 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚41934336 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17104896 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 UCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚ 6640968525710 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 G@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚59 SFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚68887040 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚ 5798803898368 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 5492441088 _step‚18 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 4552982528 E6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚321 NADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚58387316 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 123731968 LADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚894024 K>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59039232 ZHDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚ 5860666875904 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17104896 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 123731968 B9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚5237 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 5368709120 H=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚826737 SCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 58310491136 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 5492441088 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚321 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 LADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚831003 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚20643840 [KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 617940904964 þû.Ò) _wandbruntime‚67  _runtime‚67¼rºwmÓç /loss/FRt2/detection_loss‚0.7926439414239703 %loss/total_loss‚5.139324435875632 trainer/global_step‚249 _step‚19 0loss/FRt1/assignment_loss‚1.4696511736194835 0loss/FRt2/assignment_loss‚2.4321617152447375 /loss/FRt1/detection_loss‚0.4448676055874406 epoch‚0 % _timestamp‚1.7866587188355618e+09 _runtime‚ 67.365425576 êqÔë %loss/total_loss‚5.139324435875632 trainer/global_step‚249 _step‚19 0loss/FRt1/assignment_loss‚1.4696511736194835 0loss/FRt2/assignment_loss‚2.4321617152447375 /loss/FRt1/detection_loss‚0.4448676055874406 epoch‚0 % _timestamp‚1.7866587188355618e+09 _runtime‚ 67.365425576 /loss/FRt2/detection_loss‚0.7926439414239703îàã ”Õ‚(:‰ ¦ÿøÓèæšá gpu.0.gpu‚60.0 gpu.0.memory‚11, gpu.0.memoryAllocated‚7.0288848876953125) gpu.0.memoryAllocatedBytes‚ 6037766144 gpu.0.temp‚35.0 gpu.0.powerWatts‚217.39' gpu.0.enforcedPowerLimitWatts‚500.0 gpu.0.powerPercent‚43.478 gpu.0.smClock‚1410 gpu.0.memoryClock‚1593$ gpu.0.correctedMemoryErrors‚1955# gpu.0.uncorrectedMemoryErrors‚0ó^29^Ö‚(:Ó ¦ÿøÓ—ú—ä disk.sda.in‚ 161.1640625 disk.sda1.out‚0 disk.sda2.out‚0! disk.dm-0.out‚155.24951171875* disk./.usagePercent‚30.603793974311756 disk.sda.out‚ 1.3984375 disk.nvme0n1.in‚21 disk.dm-1.out‚ 1.3984375 disk.drbd2165.in‚0# proc.memory.rssMB‚ 2251.49609375) proc.memory.percent‚13.74204158782959 cpu‚22.961598173880233 network.recv‚218352$ disk./.usageGB‚456.0186462402344 disk.sda3.in‚ 161.1640625 disk.sda1.in‚0 disk.dm-1.in‚ 161.1640625 disk.dm-0.in‚21% memory_percent‚62.751221656799316) proc.memory.availableMB‚ 6102.83984375 network.sent‚363081 disk.nvme0c0n1.in‚21& disk.nvme0c0n1.out‚155.24951171875 disk.sda2.in‚0$ disk.nvme0n1.out‚155.24951171875 disk.drbd2165.out‚0 proc.cpu.threads‚17 disk.sda3.out‚ 1.3984375T+Î.×) _wandbruntime‚80  _runtime‚80݇X¬Ø¦  lr-AdamW/pg1‚ 0.00112125  lr-AdamW/pg2‚ 0.00112125 trainer/global_step‚299 $ _timestamp‚1.786658732167105e+09 _runtime‚ 80.697245939 _step‚20[f1°Ùª  lr-AdamW/pg1‚ 0.00112125  lr-AdamW/pg2‚ 0.00112125 trainer/global_step‚299 $ _timestamp‚1.786658732167105e+09 _runtime‚ 80.697245939 _step‚20Lù¬’.Ú) _wandbruntime‚80  _runtime‚80¶j5€n'ÛèN D8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚2056671 F>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 NCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚992812 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚ 7907298803712 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚4260 K@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚911956 PIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚69 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚4260 NCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚988552 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚20643840 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚321 IBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚59 J?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚988552 PCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚59036160 RGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚992812 \LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 69558198218 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 5492441088 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 123731968 UEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 69666188288 IBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚71 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 123731968 HDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4849 TBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚ 7977006923264 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 SADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚ 7976947887104 \JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚ 7002006048256 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚41347908 OCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚1068119 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 843096576 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚321 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚262 % _timestamp‚1.7866587321682189e+09 VKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚249688 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 4517085184 @7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚5237 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 5492441088 C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚4262 ]MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 73816186880 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 SGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚1068121 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 L@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚2060933 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 831258624 aQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 73860238848 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17104896 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚41931264 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 122891776 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 5368709120 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 P>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚ 7976947887104 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4262 BDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 NCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚992812 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚ 7907298803712 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚4260 K@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚911956 PIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚69 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚4260 NCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚988552 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚20643840 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚321 IBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚59 J?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚988552 PCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚59036160 RGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚992812 \LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 69558198218 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 5492441088 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 123731968 UEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 69666188288 IBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚71 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 123731968 HDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4849 TBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚ 7977006923264 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 SADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚ 7976947887104 \JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚ 7002006048256 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚41347908 OCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚1068119 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 843096576 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚321 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚262 % _timestamp‚1.7866587321682189e+09 VKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚249688 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 4517085184 @7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚5237 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 5492441088 C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚4262 ]MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 73816186880 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 SGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚1068121 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 L@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚2060933 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 831258624 aQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 73860238848 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17104896 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚41931264 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 122891776 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 5368709120 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 P>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚ 7976947887104 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4262 B5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5237 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚ 7933604036608 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚41934336 ZJDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 69939191296 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 4552982528 B6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚2063511 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚ 7933621141504 VFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 69897256960 MBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚915010 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 5368709120 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 F:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚2067779 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚59395584 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 122306414 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 123731968 D=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚59 [KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 74057261056 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 5368709120 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚321 R@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚ 8003560332800 N6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚321 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚262 Q?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚ 8003501293568 I>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚914941 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 _step‚22 E6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚321 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 SFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚70984192 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521094 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚41934336 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 5492441088 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 5492441088 H=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚991839 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 122891776 LADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚991839 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚50340352 BDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654588416 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚4266 ZJDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 69788770878 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚262 ZHDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚ 7025300783616 _ODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 74107601408 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚ 7933621141504 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17104896 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 PEDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚250399 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 4654588416 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚ 6951172538368 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 5492441088 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 PEDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚664542 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17104896 MADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚1071674 NGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚67 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 SCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 69897256960 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚223 G@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚59 LADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚996105 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 122891776 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 4552982528 J>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚2067779 VDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚ 7025229799424 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚4268 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 EDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59039232 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚4266 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 UCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚ 7961820013118 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 123731968 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚ 7892014202880 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚209 Q?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚ 7961761625802 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚41347956 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚262 IBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚59 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 843096576 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚ 7892031242240 VFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 69747422922 I=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚1071672 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 831258624 MADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚1071672 G@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚69 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 4517085184 % _timestamp‚1.7866587323422482e+09 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 123731968 trainer/global_step‚299 _runtime‚ 80.872938213 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚321 B9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚5237 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 D6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚321 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚262 Q?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚ 8003501293568 I>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚914941 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 _step‚22 E6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚321 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 SFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚70984192 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521094 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚41934336 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 5492441088 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 5492441088 H=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚991839 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 122891776 LADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚991839 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚50340352 BDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654588416 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚4266 ZJDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 69788770878 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚262 ZHDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚ 7025300783616 _ODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 74107601408 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚ 7933621141504 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17104896 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 PEDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚250399 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 4654588416 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚ 6951172538368 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 5492441088 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 PEDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚664542 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17104896 MADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚1071674 NGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚67 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 SCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 69897256960 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚223 G@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚59 LADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚996105 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 122891776 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 4552982528 J>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚2067779 VDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚ 7025229799424 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚4268 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 EDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59039232 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚4266 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 UCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚ 7961820013118 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 123731968 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚ 7892014202880 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚209 Q?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚ 7961761625802 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚41347956 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚262 IBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚59 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 843096576 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚ 7892031242240 VFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 69747422922 I=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚1071672 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 831258624 MADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚1071672 G@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚69 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 4517085184 % _timestamp‚1.7866587323422482e+09 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 123731968 trainer/global_step‚299 _runtime‚ 80.872938213 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚321 B9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚5237 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 D5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5237 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚ 7933604036608 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚41934336 ZJDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 69939191296 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 4552982528 B6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚2063511 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚ 7933621141504 VFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 69897256960 MBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚915010 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 5368709120 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 F:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚2067779 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚59395584 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 122306414 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 123731968 D=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚59àBX.à) _wandbruntime‚80  _runtime‚80úŠDÈmáç 0loss/FRt2/assignment_loss‚1.9792435932059045 %loss/total_loss‚4.753338149629946 trainer/global_step‚299 % _timestamp‚1.7866587323433988e+09 _runtime‚ 80.873466071 0loss/FRt1/assignment_loss‚1.5919776182931606 /loss/FRt1/detection_loss‚0.5424553010624235 /loss/FRt2/detection_loss‚0.6396616370684576 epoch‚0 _step‚23k¨ Tqâë 0loss/FRt1/assignment_loss‚1.5919776182931606 /loss/FRt1/detection_loss‚0.5424553010624235 /loss/FRt2/detection_loss‚0.6396616370684576 epoch‚0 _step‚23 0loss/FRt2/assignment_loss‚1.9792435932059045 %loss/total_loss‚4.753338149629946 trainer/global_step‚299 % _timestamp‚1.7866587323433988e+09 _runtime‚ 80.873466071`ß6|–ã‚(:‹ µÿøÓ˜í©á gpu.0.gpu‚60.0 gpu.0.memory‚11, gpu.0.memoryAllocated‚7.0288848876953125) gpu.0.memoryAllocatedBytes‚ 6037766144 gpu.0.temp‚35.0 gpu.0.powerWatts‚163.421' gpu.0.enforcedPowerLimitWatts‚500.0 gpu.0.powerPercent‚32.6842 gpu.0.smClock‚1410 gpu.0.memoryClock‚1593$ gpu.0.correctedMemoryErrors‚1955# gpu.0.uncorrectedMemoryErrors‚0îP Zä‚(:Ï µÿøÓùÌËã disk.sda2.in‚0 disk.nvme0n1.in‚21 disk.drbd2165.out‚0 disk.nvme0c0n1.in‚21( proc.memory.availableMB‚ 6102.0703125 cpu‚23.444409410207424 proc.cpu.threads‚17 network.sent‚436890 disk.dm-1.out‚1.828125 disk.drbd2165.in‚0& disk.nvme0c0n1.out‚164.81396484375 disk.sda.out‚1.828125 disk.sda1.in‚0$ memory_percent‚62.75591850280762# proc.memory.rssMB‚ 2251.49609375* disk./.usagePercent‚30.603835191524766% disk./.usageGB‚456.01926040649414 disk.sda3.in‚ 161.1640625 disk.sda1.out‚0 disk.sda2.out‚0$ disk.nvme0n1.out‚164.81396484375) proc.memory.percent‚13.74204158782959 network.recv‚251525 disk.dm-1.in‚ 161.1640625 disk.dm-0.in‚21! disk.dm-0.out‚164.81396484375 disk.sda3.out‚1.828125 disk.sda.in‚ 161.1640625ž¥i¥.å) _wandbruntime‚93  _runtime‚93ÌyÂa¬æ¦  lr-AdamW/pg1‚ 0.00130875  lr-AdamW/pg2‚ 0.00130875 trainer/global_step‚349 $ _timestamp‚1.786658745311543e+09 _runtime‚ 93.843993252 _step‚24 Û‡°çª _step‚24  lr-AdamW/pg1‚ 0.00130875  lr-AdamW/pg2‚ 0.00130875 trainer/global_step‚349 $ _timestamp‚1.786658745311543e+09 _runtime‚ 93.843993252†wÒù.è) _wandbruntime‚93  _runtime‚93Mº%izéîN MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 ]MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 86078013440 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 5492441088 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚ 9223398588416 SGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚1245771 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚20643840 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 PCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚59036160 OCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚1245769 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚ 8080493805568 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17104896 \JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚ 8166632320512 F?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚59 IBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚68 PCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚58387268 $ _timestamp‚1.786658745312373e+09 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚321 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚4260 K?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚1153654 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 4654588416 KDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚59 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚ 9223398588416 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 122891776 SGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚1157914 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 831258624 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 122306414 OCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚1157914 WEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚ 9304710578688 VKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚772589 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚ 8080514449408 aQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 86117871104 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚41347908 _runtime‚ 93.844130511 D8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚2399423 TBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚ 9304710578688 L@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚1063177 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 843096576 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚321 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚41931264 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 5492441088 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚321 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚ 9223415693312 XHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 81252954112 HDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 4517085184 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚321 HDeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4262 VKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚290656 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 OCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚1245771 WEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚ 9256255134330 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 5492441088 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚4260 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 PDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚1063245 \LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 81120008826 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚262 IBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚59 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚ 9175118086144 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 5368709120 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 \LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 81294885376 L@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚2403685 RGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚772587 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 BDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚262 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚262 @7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚5237 UEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 81252954112 G>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4849 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚ 9175135125504 _step‚25 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚41931264 XFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚ 8166571819008 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚59395584 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 OCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚1153654 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 123731968 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚ 9223415693312 UHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚60501504 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚39857664 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 123731968 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 123731968 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚429 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 4628521094 D;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚5237 YIDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 81294885376 P>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚ 9304651542528 PIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚66 SADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚ 9256196747062 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 5368709120øµã]x'êòN HDeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4262 VKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚290656 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 OCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚1245771 WEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚ 9256255134330 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 5492441088 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚4260 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 PDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚1063245 \LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 81120008826 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚262 IBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚59 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚ 9175118086144 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 5368709120 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 \LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 81294885376 L@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚2403685 RGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚772587 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 BDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚262 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚262 @7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚5237 UEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 81252954112 G>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4849 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚ 9175135125504 _step‚25 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚41931264 XFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚ 8166571819008 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚59395584 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 OCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚1153654 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 123731968 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚ 9223415693312 UHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚60501504 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚39857664 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 123731968 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 123731968 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚429 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 4628521094 D;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚5237 YIDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 81294885376 P>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚ 9304651542528 PIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚66 SADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚ 9256196747062 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 5368709120 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 ]MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 86078013440 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 5492441088 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚ 9223398588416 SGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚1245771 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚20643840 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 PCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚59036160 OCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚1245769 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚ 8080493805568 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17104896 \JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚ 8166632320512 F?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚59 IBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚68 PCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚58387268 $ _timestamp‚1.786658745312373e+09 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚321 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚4260 K?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚1153654 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 4654588416 KDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚59 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚ 9223398588416 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 122891776 SGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚1157914 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 831258624 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 122306414 OCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚1157914 WEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚ 9304710578688 VKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚772589 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚ 8080514449408 aQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 86117871104 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚41347908 _runtime‚ 93.844130511 D8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚2399423 TBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚ 9304710578688 L@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚1063177 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 843096576 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚321 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚41931264 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 5492441088 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚321 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚ 9223415693312 XHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 81252954112 HDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 4517085184 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚321Ô–S¹.ë) _wandbruntime‚94  _runtime‚94x:—|&ìùL EDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654588416 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17104896 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚48243200 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 MADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚1249322 PEDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚774792 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚ 8103541178368 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚59395584 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 5368709120 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚321 J>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚2410531 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521094 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚262 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚4266 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚41934336 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚ 9249720926208 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 122306414 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚321 NADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚58387316 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚321 B6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚2406263 SCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 81484022784 Q?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚ 9331204948992 _ODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 86370566656 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 TIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚774794 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 trainer/global_step‚349 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚ 9201320919040 ZJDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 81350581486 I=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚1156941 PEDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚291408 BDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚1066200 TIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚291479 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚20643840 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 IBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚59 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚209 ZHDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚ 8189932388864 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 EDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 122891776 G@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚73 >5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5237 I=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚1249322 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 843096576 K>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59039232 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚ 9249738031104 I@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚4849 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚321 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 4552982528 NBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚1066273 NGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚71 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0ó[¾G¾íýL TIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚774794 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 trainer/global_step‚349 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚ 9201320919040 ZJDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 81350581486 I=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚1156941 PEDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚291408 BDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚1066200 TIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚291479 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚20643840 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 IBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚59 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚209 ZHDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚ 8189932388864 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 EDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 122891776 G@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚73 >5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5237 I=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚1249322 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 843096576 K>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59039232 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚ 9249738031104 I@DeviceStatsMonitor.on_train_batch_eÚHC%Å nd/allocation.small_pool.peak‚4849 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚321 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 4552982528 NBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚1066273 NGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚71 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 EDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654588416 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17104896 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚48243200 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 MADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚1249322 PEDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚774792 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚ 8103541178368 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚59395584 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 5368709120 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚321 J>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚2410531 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521094 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚262 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚4266 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚41934336 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚ 9249720926208 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 122306414 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚321 NADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚58387316 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚321 B6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚2406263 SCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 81484022784 Q?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚ 9331204948992 _ODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 86370566656 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0kù.î) _wandbruntime‚94  _runtime‚94žF,lïæ /loss/FRt2/detection_loss‚0.7095360149898501 %loss/total_loss‚4.860249604343634 epoch‚0 trainer/global_step‚349 _runtime‚ 94.038689103 0loss/FRt2/assignment_loss‚1.9789444635467557 % _timestamp‚1.7866587455087225e+09 _step‚27 /loss/FRt1/assignment_loss‚1.675288721911686 /loss/FRt1/detection_loss‚0.4964804038953421dKsUpðê /loss/FRt1/detection_loss‚0.4964804038953421 /loss/FRt2/detection_loss‚0.7095360149898501 %loss/total_loss‚4.860249604343634 epoch‚0 trainer/global_step‚349 _runtime‚ 94.038689103 0loss/FRt2/assignment_loss‚1.9789444635467557 % _timestamp‚1.7866587455087225e+09 _step‚27 /loss/FRt1/assignment_loss‚1.675288721911686˜5´ ñ‚(:• ÄÿøÓá gpu.0.gpu‚17.0 gpu.0.memory‚2, gpu.0.memoryAllocated‚7.0288848876953125) gpu.0.memoryAllocatedBytes‚ 6037766144 gpu.0.temp‚35.0 gpu.0.powerWatts‚204.746' gpu.0.enforcedPowerLimitWatts‚500.0) gpu.0.powerPercent‚40.949200000000005 gpu.0.smClock‚1410 gpu.0.memoryClock‚1593$ gpu.0.correctedMemoryErrors‚1955# gpu.0.uncorrectedMemoryErrors‚0ýŠEnò‚(:ã ÄÿøÓ£¸Âã disk.sda3.in‚ 182.15234375& disk.nvme0c0n1.out‚210.09033203125 disk.sda1.in‚0 disk.sda1.out‚0 disk.sda2.out‚0! disk.dm-0.out‚210.09033203125 cpu‚23.82686295355416) disk./.usagePercent‚30.60384184772066% disk./.usageGB‚456.01935958862305 disk.sda3.out‚ 1.98828125 disk.nvme0c0n1.in‚ 32.578125 disk.sda.in‚ 182.15234375 disk.dm-1.in‚ 182.15234375$ memory_percent‚62.75649070739746 proc.memory.rssMB‚2251.5 network.sent‚510711 network.recv‚284333 disk.sda2.in‚0$ disk.nvme0n1.out‚210.09033203125 proc.cpu.threads‚17 disk.sda.out‚ 1.98828125 disk.nvme0n1.in‚ 32.578125 disk.dm-1.out‚ 1.98828125 disk.drbd2165.in‚0 disk.drbd2165.out‚0 disk.dm-0.in‚ 32.578125( proc.memory.availableMB‚ 6101.9765625( proc.memory.percent‚13.7420654296875±ÃE”0ó+ _wandbruntime‚106  _runtime‚106ª^®ßÄô¾ & lr-AdamW/pg1‚0.0014962500000000002 & lr-AdamW/pg2‚0.0014962500000000002 trainer/global_step‚399 % _timestamp‚1.7866587584240243e+09 _runtime‚ 106.954085023 _step‚28@¯ÅÈõ & lr-AdamW/pg1‚0.0014962500000000002 & lr-AdamW/pg2‚0.0014962500000000002 trainer/global_step‚399 % _timestamp‚1.7866587584240243e+09 _runtime‚ 106.954085023 _step‚28㩃0ö+ _wandbruntime‚106  _runtime‚106›¦åN‚'÷üN NFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚223 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚48246272 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 KBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚4849 _runtime‚ 106.958924922 HDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚10632355197952 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚4260 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 [HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚10484407762944 M@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚59036160 UEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 92839719936 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 831258624 OCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚1423421 _LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚10539532582912 JBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚665 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 5368709120 XEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚10539515478016 RGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚882837 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 122891776 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚20643840 XEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚10577106621738 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 4654588416 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚321 % _timestamp‚1.7866587584250062e+09 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4262 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚209 ]MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 98349742592 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17104896 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 VKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚882839 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 123731968 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚262 HDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 VKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚332366 RGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚332295 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚262 @7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚5237 \LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 92881651200 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 122891776 PDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚1215205 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 843096576 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚ 9232862445568 \LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 92681819434 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 4552982528 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 4628521094 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚321 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚262 G>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4849 YIDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 92881651200 XFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚ 9331212188160 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚41347908 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 123731968 D8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚2742175 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17104896 [HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚10539515478016 SGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚1323016 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚321 _step‚29 PCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚59036160 L@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚2746437 D;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚5237 BDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 VKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚332366 RGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚332295 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚262 @7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚5237 \LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 92881651200 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 122891776 PDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚1215205 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 843096576 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚ 9232862445568 \LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 92681819434 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 4552982528 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 4628521094 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚321 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚262 G>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4849 YIDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 92881651200 XFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚ 9331212188160 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚41347908 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 123731968 D8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚2742175 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17104896 [HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚10539515478016 SGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚1323016 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚321 _step‚29 PCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚59036160 L@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚2746437 D;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚5237 BDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚10632355197952 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚4260 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 [HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚10484407762944 M@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚59036160 UEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 92839719936 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 831258624 OCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚1423421 _LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚10539532582912 JBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚665 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 5368709120 XEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚10539515478016 RGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚882837 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 122891776 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚20643840 XEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚10577106621738 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 4654588416 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚321 % _timestamp‚1.7866587584250062e+09 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4262 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚209 ]MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 98349742592 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚170393602x0ù+ _wandbruntime‚107  _runtime‚107ƒÑãú†M WGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 93112722944 QEDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚1326309 PEDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚333122 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 5492441088 _runtime‚ 107.121306585 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 B6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚321 % _timestamp‚1.7866587585909586e+09 I=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚1322043 ODeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚1218164 VCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚10603522987934 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 123731968 F:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚2753283 E5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5237 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17104896 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 4552982528 QEDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚1426974 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 D=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚59 D6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚321 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚223 VDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚ 9354507573760 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 ]JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚10510610595840 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 5368709120 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17104896 J>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚2753283 G@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚59 K>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59039232 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 4654588416 YFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚10565837815808 _step‚30 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚4268 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚4266 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚ 9255909818368 _ODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 98635512832 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 4517085184 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 5492441088 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚262 B6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚2749015 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 R?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚10658908604416 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654588416 VFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 93070788608 B9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚5237 NADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚58387316 VCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚10565837815808 SCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 93070788608 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚41934336 VCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚10658967643648 ZHDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚ 9354565975040 SFDeviceStaŽIµï tsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚58401280 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚37757440 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 MADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚1426974 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚429 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚262 R?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚10603464600618 TIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚885044 ZGDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚10565854920704 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 122891776 MADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚1322043 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 4552982528 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 122891776 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚209 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚20643840 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 YFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚10510593556480 trainer/global_step‚399 F:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚2749015 TIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚333181 ZJDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 92912392094 VFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 92871044138 I@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚4849 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 843096576 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 831258624 NADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚59039232 [KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 98597755392 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521094 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 I=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚1426972 EDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 R?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚10658908604416 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654588416 VFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 93070788608 B9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚5237 NADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚58387316 VCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚10565837815808 SCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 93070788608 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚41934336 VCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚10658967643648 ZHDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚ 9354565975040 SFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚58401280 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚37757440 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 MADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚1426974 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚429 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚262 R?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚10603464600618 TIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚885044 ZGDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚10565854920704 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 122891776 MADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚1322043 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 4552982528 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 122891776 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚209 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚20643840 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 YFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚10510593556480 trainer/global_step‚399 F:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚2749015 TIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚333181 ZJDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 92912392094 VFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 92871044138 I@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚4849 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 843096576 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 831258624 NADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚59039232 [KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 98597755392 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521094 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 I=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚1426972 E6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚321 % _timestamp‚1.7866587585909586e+09 I=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚1322043 ODeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚1218164 VCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚10603522987934 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 123731968 F:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚2753283 E5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5237 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17104896 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 4552982528 QEDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚1426974 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 D=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚59 D6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚321 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚223 VDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚ 9354507573760 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 ]JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚10510610595840 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 5368709120 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17104896 J>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚2753283 G@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚59 K>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59039232 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 4654588416 YFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚10565837815808 _step‚30 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚4268 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚4266 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚ 9255909818368 _ODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 98635512832 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 4517085184¯” 0ü+ _wandbruntime‚107  _runtime‚1075 nýè /loss/FRt1/detection_loss‚0.4614024698948649 /loss/FRt2/detection_loss‚0.7559808987965683 epoch‚0 trainer/global_step‚399 _runtime‚ 107.121893404 0loss/FRt1/assignment_loss‚1.5793822800706112 %loss/total_loss‚4.448795796015878 % _timestamp‚1.7866587585920827e+09 _step‚31 0loss/FRt2/assignment_loss‚1.6520301472538335ãØorþì _step‚31 0loss/FRt2/assignment_loss‚1.6520301472538335 /loss/FRt1/detection_loss‚0.4614024698948649 /loss/FRt2/detection_loss‚0.7559808987965683 epoch‚0 trainer/global_step‚399 _runtime‚ 107.121893404 0loss/FRt1/assignment_loss‚1.5793822800706112 %loss/total_loss‚4.448795796015878 % _timestamp‚1.7866587585920827e+09r*¨ë;ÿb6 jet/accuracy_2_of_2"trainer/global_step2:JPg÷94@€b; particle/accuracy_2_of_2"trainer/global_step2:JP¬Ia<b7 particle/sensitivity"trainer/global_step2:JPwÒ8‚b3 particle/f_score"trainer/global_step2:JP»†úJ@ƒb; Purity/*FRt/event_purity"trainer/global_step2:JP û?#>„b9 Purity/*FRt/FRt_purity"trainer/global_step2:JPÓ m;…b6 jet/accuracy_1_of_2"trainer/global_step2:JPO`þ²@†b; particle/accuracy_1_of_1"trainer/global_step2:JP‹oæP@‡b; Purity/1FRt/event_purity"trainer/global_step2:JP´¥`©Dˆb? Purity/1FRt/event_proportion"trainer/global_step2:JP6»Ï&D‰b? Purity/2FRt/event_proportion"trainer/global_step2:JPQ—k÷>Šb9 Purity/2FRt/FRt_purity"trainer/global_step2:JPù®£";‹b6 jet/accuracy_1_of_1"trainer/global_step2:JPŠ¹G›9Œb4 particle/accuracy"trainer/global_step2:JPRp¥Æ@b; particle/accuracy_1_of_2"trainer/global_step2:JP&u¿@Žb; Purity/2FRt/event_purity"trainer/global_step2:JP?¦2D;b6 validation_accuracy"trainer/global_step2:JPºpºGbB validation_average_jet_accuracy"trainer/global_step2:JP{KWc;‘b6 loss/val_total_loss"trainer/global_step2:JPx=aD’b? Purity/0FRt/event_proportion"trainer/global_step2:JPþ?úò<“b7 particle/specificity"trainer/global_step2:JPx¥ÿ>”b9 Purity/1FRt/FRt_purity"trainer/global_step2:JPΩ&aD•b? Purity/*FRt/event_proportion"trainer/global_step2:JP¡¢@–b; Purity/0FRt/event_purity"trainer/global_step2:JPuF½Á0—+ _wandbruntime‚112  _runtime‚112séYíߘ٠0particle/accuracy_2_of_2‚0.12300430983304977 ,particle/sensitivity‚0.26375430822372437 'particle/f_score‚0.3472287952899933 /Purity/*FRt/event_purity‚0.1695062816143036 .Purity/*FRt/FRt_purity‚0.17794165015220642 +jet/accuracy_1_of_2‚0.26211947202682495 particle/accuracy_1_of_1‚1 0Purity/1FRt/event_purity‚0.17328743636608124 3Purity/1FRt/event_proportion‚0.6297432780265808 2Purity/2FRt/event_proportion‚0.040062315762043 -Purity/2FRt/FRt_purity‚0.2139391005039215 +jet/accuracy_1_of_1‚0.11993423104286194 (particle/accuracy‚0.6480887532234192 epoch‚0 .particle/accuracy_1_of_2‚0.548137903213501 % _timestamp‚1.7866587639972305e+09 _step‚32 0Purity/2FRt/event_purity‚0.10946358740329742 ,validation_accuracy‚0.057475294917821884 7validation_average_jet_accuracy‚0.12235353887081146 _runtime‚ 112.5273982 )loss/val_total_loss‚4.470885702517972 3Purity/0FRt/event_proportion‚0.3301943838596344 +particle/specificity‚0.8595503568649292 trainer/global_step‚399 .Purity/1FRt/FRt_purity‚0.17328743636608124 2Purity/*FRt/event_proportion‚0.669805645942688 Purity/0FRt/event_purity‚1 ,jet/accuracy_2_of_2‚0.057475294917821884ì•ã™Ý /Purity/*FRt/event_purity‚0.1695062816143036 .Purity/*FRt/FRt_purity‚0.17794165015220642 +jet/accuracy_1_of_2‚0.26211947202682495 particle/accuracy_1_of_1‚1 0Purity/1FRt/event_purity‚0.17328743636608124 3Purity/1FRt/event_proportion‚0.6297432780265808 2Purity/2FRt/event_proportion‚0.040062315762043 -Purity/2FRt/FRt_purity‚0.2139391005039215 +jet/accuracy_1_of_1‚0.11993423104286194 (particle/accuracy‚0.6480887532234192 epoch‚0 .particle/accuracy_1_of_2‚0.548137903213501 % _timestamp‚1.7866587639972305e+09 _step‚32 0Purity/2FRt/event_purity‚0.10946358740329742 ,validation_accuracy‚0.057475294917821884 7validation_average_jet_accuracy‚0.12235353887081146 _runtime‚ 112.5273982 )loss/val_total_loss‚4.470885702517972 3Purity/0FRt/event_proportion‚0.3301943838596344 +particle/specificity‚0.8595503568649292 trainer/global_step‚399 .Purity/1FRt/FRt_purity‚0.17328743636608124 2Purity/*FRt/event_proportion‚0.669805645942688 Purity/0FRt/event_purity‚1 ,jet/accuracy_2_of_2‚0.057475294917821884 0particle/accuracy_2_of_2‚0.12300430983304977 ,particle/sensitivity‚0.26375430822372437 'particle/f_score‚0.3472287952899933 CØèD¡š‚(:– ÓÿøÓìÇ›á gpu.0.gpu‚64.0 gpu.0.memory‚12, gpu.0.memoryAllocated‚7.0288848876953125) gpu.0.memoryAllocatedBytes‚ 6037766144 gpu.0.temp‚33.0 gpu.0.powerWatts‚133.111' gpu.0.enforcedPowerLimitWatts‚500.0) gpu.0.powerPercent‚26.622199999999996 gpu.0.smClock‚1410 gpu.0.memoryClock‚1593$ gpu.0.correctedMemoryErrors‚1955# gpu.0.uncorrectedMemoryErrors‚0ž¾{›‚(:ð ÓÿøÓ–Âÿã* disk./.usagePercent‚30.607830189096585 disk.nvme0n1.in‚ 34.01171875 disk.dm-0.in‚ 34.01171875! disk.nvme0c0n1.in‚ 34.01171875$ disk.nvme0c0n1.out‚ 277.181640625# memory_percent‚63.3373498916626# proc.memory.rssMB‚ 2285.77734375 network.recv‚311067 disk.dm-1.in‚ 1174.91015625 disk.dm-0.out‚ 277.181640625 disk.sda1.out‚0) proc.memory.availableMB‚ 6006.80859375* proc.memory.percent‚13.951277732849121 cpu‚24.290475211279844 proc.cpu.threads‚17" disk.nvme0n1.out‚ 277.181640625 disk.drbd2165.in‚0 disk.drbd2165.out‚0 disk.sda.out‚ 2.30078125 disk.sda1.in‚0 disk.sda2.in‚0 disk.sda2.out‚0 network.sent‚578605$ disk./.usageGB‚456.0787887573242 disk.dm-1.out‚ 2.30078125 disk.sda3.in‚ 1174.91015625 disk.sda3.out‚ 2.30078125 disk.sda.in‚ 1174.91015625‚Ûí0œ+ _wandbruntime‚126  _runtime‚126´ö¤Ã½ & lr-AdamW/pg2‚0.0014999768682335923 trainer/global_step‚449 % _timestamp‚1.7866587776554666e+09 _runtime‚ 126.18552204 _step‚33 & lr-AdamW/pg1‚0.0014999768682335923›Ç'ÇžÁ % _timestamp‚1.7866587776554666e+09 _runtime‚ 126.18552204 _step‚33 & lr-AdamW/pg1‚0.0014999768682335923 & lr-AdamW/pg2‚0.0014999768682335923 trainer/global_step‚449!-ÆV0Ÿ+ _wandbruntime‚126  _runtime‚126›â™Ž' ˆO VEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 108494196736 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚262 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚59395584 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 123731968 [HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚12246569582592 YHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 108494196736 OCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚1661675 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚1033703 ]LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 108309602143 G>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4874 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 5368709120 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 123731968 _step‚34 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 ZIDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 108536140800 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 4552982528 L@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚3171982 RGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚382495 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 5492441088 F>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 PDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚1416283 ]JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚10799295691776 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 OCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚1661677 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 4654601216 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17104896 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚54524928 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 5492441088 trainer/global_step‚449 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 5368709120 ]LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 108536140800 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 UHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚75168768 IBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚87 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚223 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚262 K?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚1661675 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17104896 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚429 dQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚10684522037248 F?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚59 BDeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4287 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚262 Q>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚12355063779328 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 4517085184 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 5368709120 D8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚3167695 \IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚12246586687488 OCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚1506020 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 PCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚59048960 D;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚5262 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚209 YFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚10799220523008 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1ë ÿò ¡ŒO PCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚59048960 D;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚5262 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚209 YFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚10799220523008 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 VEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 108494196736 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚262 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚59395584 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 123731968 [HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚12246569582592 YHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 108494196736 OCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚1661675 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚1033703 ]LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 108309602143 G>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4874 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 5368709120 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 123731968 _step‚34 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 ZIDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 108536140800 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 4552982528 L@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚3171982 RGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚382495 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 5492441088 F>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 PDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚1416283 ]JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚10799295691776 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 OCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚1661677 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 4654601216 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17104896 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚54524928 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 5492441088 trainer/global_step‚449 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 5NIF\ 368709120 ]LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 108536140800 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 UHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚75168768 IBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚87 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚223 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚262 K?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚1661675 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17104896 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚429 dQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚10684522037248 F?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚59 BDeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4287 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚262 Q>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚12355063779328 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 4517085184 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 5368709120 D8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚3167695 \IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚12246586687488 OCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚1506020 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0"J¾ß0¢+ _wandbruntime‚126  _runtime‚126àQUx™&£“M F:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚3178828 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 4552982528 NGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚77 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚321 MADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚1665230 EDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654601216 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚1035906 TIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚383396 _step‚35 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 4552982528 QEDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚1665230 MADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚1665228 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚209 ^KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚10707548766208 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 122306518 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17104896 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚59395584 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521198 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 B6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚3174535 [JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 108767212544 B6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚321 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚429 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 5492441088 G@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚59 ODeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚3178828 S@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚12381676237824 MADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚1509307 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚223 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 831258624 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚262 QEDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚1513598 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 R?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚12309281551863 [JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 108540174803 NBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚1419304 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 B9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚5262 DDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 ZGDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚12272909025280 E8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 5368709120 VCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚12272891920384 R?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚12381617185792 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 843096576 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚50327552 I=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚1665228 MADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚1513598 VCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚12309339939283 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 5492441088 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 >5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5262 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 YFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚12272891920384 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 XGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 108767212544 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 122904576 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 123731968 VCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚12381676237824 ]JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚12272909025280 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚41947136 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚20643840 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 123731968 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚321 _runtime‚ 126.35444685 TCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 108725265408 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 5492441088 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚262 K>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59052032 J>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚1419225 SFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚70971392 bODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚10707569410048 \KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 114966421504 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 5368709120 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 YFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚12200782725120 IBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚59 trainer/global_step‚449Vu±r&¤—M ]JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚12200799764480 D=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚59 % _timestamp‚1.7866587778240545e+09 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17104896 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 122904576 I@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚4874 WDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚10822515187712 `ODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 115016749056 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚41348060 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 5368709120 I=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚1509307 H@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚665 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚4291 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚41947136 G@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚79 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚1035908 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 123731968 NADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚59052032 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 4654601216 J>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚3178828 S@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚12381676237824 MADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚1509307 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚223 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 831258624 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚262 QEDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚1513598 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 R?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚12309281551863 [JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 108540174803 NBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚1419304 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 B9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚5262 DDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 ZGDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚12272909025280 E8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 5368709120 VCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚12272891920384 R?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚12381617185792 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 843096576 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚50327552 I=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚1665228 MADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚1513598 VCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚12309339939283 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 5492441088 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 >5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5262 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 YFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚12272891920384 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 XGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 108767212544 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 122904576 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 123731968 VCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚12381676237824 ]JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚12272909025280 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚41947136 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚20643840 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 123731968 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚321 _runtime‚ 126.35444685 TCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 108725265408 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 5492441088 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚262 K>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59052032 J>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚1419225 SFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚70971392 bODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚10707569410048 \KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 114966421504 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 5368709120 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 YFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚12200782725120 IBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚59 trainer/global_step‚449 F:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚3178828 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 4552982528 NGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚77 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚321 MADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚1665230 EDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654601216 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚1035906 TIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚383396 _step‚35 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 4552982528 QEDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚1665230 MADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚1665228 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚209 ^KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚10707548766208 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 122306518 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17104896 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚59395584 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521198 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 B6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚3174535 [JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 108767212544 B6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚321 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚429 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 5492441088 G@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚59 ODeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚13682767434752 VEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 120080962560 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 122904576 ]LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 120122906624 L@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚1566865 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚20643840 KDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚59 L@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚3514734 SGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚1839327 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚429 PIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚64 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 _runtime‚ 138.835221315 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 4517085184 K?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚1671122 YFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚11963848126464 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 831258624 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚41348012 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 122306518 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 4628521198 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚262 K?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚1839325 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚209 PCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚58387372 trainer/global_step‚499 OCDev©7OiceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚1839327 KBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚4874 IBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚66 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚262 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚1143953 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 5492441088 ]JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚11963914906624 D8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚3510447 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚321 XEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚13682826483712 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 IBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚59 ZIDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 120122906624 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4287 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 F?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚59 _step‚38 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚4285 OCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚1671122 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 5368709120 PCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚59048960 dQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚11836890677248 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚321 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚321 JBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚665 UBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚13682826483712 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 D;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚5262 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚223 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 4654601216 UHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚66780160 ]LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 119871412751 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 5368709120 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17104896 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 4552982528 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 HDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚4287 G>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4874 \IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚13562703577088 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚41944064 RGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚422914 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚59395584 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚4285 [HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚13562686472192 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 843096576 BDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 XEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚13562686472192 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 OCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚1839325ìÅ”'¯ŽO SGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚1675407 PDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚1566931 `MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚11836870033408 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚46136320 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚262 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 _LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚13562703577088 XEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚13603775060495 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 123731968 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 123731968 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 4654601216 F>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 XEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚13562686472192 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 OCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚1839325 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 5492441088 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚41944064 TADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚13682767434752 HDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚13682767434752 VEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 120080962560 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 122904576 ]LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 120122906624 L@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚1566865 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚20643840 KDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚59 L@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚3514734 SGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚1839327 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚429 PIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚64 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 _runtime‚ 138.835221315 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 4517085184 K?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚1671122 YFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚11963848126464 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 831258624 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚41348012 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 122306518 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 4628521198 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚262 K?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚1839325 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚209 PCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚58387372 trainer/global_step‚499 OCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚1839327 KBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚4874 IBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚66 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚262 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚1143953 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 5492441088 ]JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚11963914906624 D8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚3510447 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚321 XEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚13682826483712 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 IBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚59 ZIDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 120122906624 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4287 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 F?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚59 _step‚38 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚4285 OCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚1671122 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 5368709120 PCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚59048960 dQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚11836890677248 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚321 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚321 JBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚665 UBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚13682826483712 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 D;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚5262 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚223 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 4654601216 UHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚66780160 ]LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 119871412751 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 5368709120 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17104896 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 4552982528 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 HDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚4287 G>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4874 \IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚13562703577088 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚41944064 RGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚422914 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚59395584 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚4285 [HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚13562686472192 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 843096576 BDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654601216 B9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚5262 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚209 bODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚11859938050048 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 123731968 R?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚13709320841216 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚50327552 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 VCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚13589008809984 SFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚70971392 `ODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 127271682048 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚321 G@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚59 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 5492441088 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚321 trainer/global_step‚499 NGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚76 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚321 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 5492441088 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 122904576 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 831258624 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚321 I=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚1842878 EDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚1569902 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚223 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 NADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚58387420 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 5368709120 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 4552982528 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚262 _step‚39 G@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚78 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 123731968 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 D5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5262 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 VCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚13630191426691 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 PEDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚423746 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚59395584 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 5368709120 K>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59052032 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚262 YFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚13589008809984 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚1146158 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521198 YFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚13510072401920 F:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚3521580 B6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚3517287 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 BDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 VCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚13709379893248 ODeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚3521580 [HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚11987209732096 ]JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚13510089441280 MADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚1842878 ^KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚11859917406208 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 5368709120 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚4293 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17104896 ]JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚13589025914880 NADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚59052032 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 4654601216 XGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 120353978368 EDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 D=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚59ú:7»z²˜M WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚59395584 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 5368709120 K>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59052032 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚262 YFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚13589008809984 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚1146158 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521198 YFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚13510072401920 F:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚3521580 B6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚3517287 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 BDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 VCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚13709379893248 ODeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚3521580 [HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚11987209732096 ]JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚13510089441280 MADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚1842878 ^KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚11859917406208 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 5368709120 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚4293 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17104896 ]JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚13589025914880 NADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚59052032 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 4654601216 XGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 120353978368 EDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 D=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚59 QEDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚1678700 % _timestamp‚1.7866587904643526e+09 MADeviceStatsMonitor.on_train_batch_end/active.large_pool.allo•ä›4$cated‚1842880 NBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚1569980 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 123731968 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654601216 B9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚5262 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚209 bODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚11859938050048 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 123731968 R?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚13709320841216 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚50327552 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 VCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚13589008809984 SFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚70971392 `ODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 127271682048 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚321 G@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚59 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 5492441088 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚321 trainer/global_step‚499 NGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚76 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚321 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 5492441088 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 122904576 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 831258624 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚321 I=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚1842878 EDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚1569902 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚223 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 NADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚58387420 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 5368709120 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 4552982528 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚262 _step‚39 G@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚78 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 123731968 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 D5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5262 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 VCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚13630191426691 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 PEDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚423746 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0'îÙ[˜0³+ _wandbruntime‚138  _runtime‚138FÆxšm´ç /loss/FRt1/assignment_loss‚1.584064448690154 /loss/FRt2/detection_loss‚0.6847319333429299 % _timestamp‚1.7866587904652588e+09 _runtime‚ 138.995063829 _step‚40 /loss/FRt2/assignment_loss‚1.643898127140392 0loss/FRt1/detection_loss‚0.45538464722449845 %loss/total_loss‚4.368079156397974 epoch‚1 trainer/global_step‚499ÏØíqµë /loss/FRt2/assignment_loss‚1.643898127140392 0loss/FRt1/detection_loss‚0.45538464722449845 %loss/total_loss‚4.368079156397974 epoch‚1 trainer/global_step‚499 /loss/FRt1/assignment_loss‚1.584064448690154 /loss/FRt2/detection_loss‚0.6847319333429299 % _timestamp‚1.7866587904652588e+09 _runtime‚ 138.995063829 _step‚40(ëÓ)‹•¶‚(:Š ñÿøÓöþøà gpu.0.gpu‚40.0 gpu.0.memory‚9, gpu.0.memoryAllocated‚7.0288848876953125) gpu.0.memoryAllocatedBytes‚ 6037766144 gpu.0.temp‚33.0 gpu.0.powerWatts‚119.593' gpu.0.enforcedPowerLimitWatts‚500.0 gpu.0.powerPercent‚23.9186 gpu.0.smClock‚1410 gpu.0.memoryClock‚1593$ gpu.0.correctedMemoryErrors‚1955# gpu.0.uncorrectedMemoryErrors‚0Ü(£—u·‚(:ê ñÿøÓÆ×€ã disk.drbd2165.in‚0 disk.drbd2165.out‚0 disk.dm-0.out‚477.7099609375) proc.memory.availableMB‚ 6005.91015625% disk./.usageGB‚455.54630279541016 disk.nvme0c0n1.in‚40.65625 disk.sda1.in‚0 disk.sda2.in‚0 disk.dm-1.in‚ 1224.7265625 cpu‚24.737273890775334 proc.cpu.threads‚17 network.sent‚728848% disk.nvme0c0n1.out‚477.7099609375 disk.nvme0n1.in‚40.65625 disk.dm-1.out‚ 2.73046875# proc.memory.rssMB‚ 2285.77734375* proc.memory.percent‚13.951277732849121 network.recv‚375549 disk.sda.in‚ 1224.7265625 disk.sda.out‚ 2.73046875 disk.sda1.out‚0# disk.nvme0n1.out‚477.7099609375 disk.dm-0.in‚40.65625% memory_percent‚63.342833518981934* disk./.usagePercent‚30.572094609407053 disk.sda3.in‚ 1224.7265625 disk.sda3.out‚ 2.73046875 disk.sda2.out‚0ÁŒ«0¸+ _wandbruntime‚151  _runtime‚151\ÎL¹¼ % _timestamp‚1.7866588027316766e+09 _runtime‚ 151.261679681 _step‚41 % lr-AdamW/pg1‚0.001499786119709853 % lr-AdamW/pg2‚0.001499786119709853 trainer/global_step‚549“›ÂjƺÀ % lr-AdamW/pg2‚0.001499786119709853 trainer/global_step‚549 % _timestamp‚1.7866588027316766e+09 _runtime‚ 151.261679681 _step‚41 % lr-AdamW/pg1‚0.001499786119709853)n‘—0»+ _wandbruntime‚151  _runtime‚151ðØty'¼ŠO \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚41942016 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 5492441088 dQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚12989259317248 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 4517085184 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚429 ^MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 139241639936 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚41944064 SGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚2016977 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 OCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚1840509 IBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚68 YFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚13128480313344 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚321 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 PCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚58387372 ]LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 131433223359 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 5368709120 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 122306518 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 L@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚1717712 \IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚14878820466688 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚59395584 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚41348012 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚262 C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚4287 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚41944064 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 123731968 % _timestamp‚1.7866588027326446e+09 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚262 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 trainer/global_step‚549 _runtime‚ 151.265893415 YHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 131391875347 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚4285 VEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 131667728384 ]LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 131709672448 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4287 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚321 PCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚59048960 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚20643840 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 M@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚59048960 PIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚66 UBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚15010530139136 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17104896 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 122904576 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚1254201 [HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚14793176285184 SGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚1840509 ]JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚13128542899200 IBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚59 VKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚463577 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 5492441088 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 _LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚14793193324544 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 123731968 D;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚5262 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚262 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 4654601216 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 BDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 KBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚4874 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 831258624 F>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 ZIDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 131709672448 `MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚12989238673408 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 123731968 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚1254203 K?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚2016975 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 122904576 PDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚1717780 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 4654601216 KDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚59 OCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚1836224 bQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 139283581952 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 5368709120 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚321 F?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚59 TADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚15010471090176 XEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚15010530139136 _LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚14878820466688 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 843096576 Q>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚15010471090176 L@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚3857486 HDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4874 [HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚14878803361792 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 5492441088I-@Y”'½ŽO OCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚2016975 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚321 OCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚2016977 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚209 UHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚62585856 XEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚14924626547903 TADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚14924568160531 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 5368709120 @7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚5262 K?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚1836224 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 4552982528 YHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 131667728384 D8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚3853199 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 4552982528 RGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚463511 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 KBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚4874 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 831258624 F>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 ZIDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 131709672448 `MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚12989238673408 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 123731968 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚1254203 K?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚2016975 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 122904576 PDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚1717780 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 4654601216 KDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚59 OCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚1836224 bQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 139283581952 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 5368709120 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚321 F?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚59 TADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚15010471090176 XEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚15010530139136 _LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚14878820466688 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 843096576 Q>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚15010471090176 L@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚3857486 HDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4874 [HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚14878803361792 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 5492441088 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚41942016 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 5492441088 dQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚12989259317248 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 4517085184 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚429 ^MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 139241639936 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚41944064 SGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚2016977 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 OCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚1840509 IBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚68 YFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚13128480313344 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚321 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 PCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚58387372 ]LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 131433223359 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 5368709120 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 122306518 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 L@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚1717712 \IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚14878820466688 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚59395584 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚41348012 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚262 C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚4287 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚41944064 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 123731968 % _timestamp‚1.7866588027326446e+09 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚262 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 trainer/global_step‚549 _runtime‚ 151.265893415 YHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 131391875347 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚4285 VEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 131667728384 ]LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 131709672448 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4287 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚321 PCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚59048960 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚20643840 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 M@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚59048960 PIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚66 UBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚15010530139136 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17104896 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 122904576 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚1254201 [HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚14793176285184 SGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚1840509 ]JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚13128542899200 IBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚59 VKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚463577 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 5492441088 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 _LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚14793193324544 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 123731968 D;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚5262 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚262 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 4654601216 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 B5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5262 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17104896 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 4517085184 [JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 131663796019 NGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚75 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 831258624 B9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚5262 TIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚464395 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 123731968 DDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚3864332 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521198 TCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 131898797056 EDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 NBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚1720803 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 R?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚14950984526679 ]JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚14819379118080 WFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 131898797056 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 843096576 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚4291 K>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59052032 [JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 131940744192 QEDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚2020530 WFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 131622447959 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚262 VCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚15037083548672 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 123731968 _runtime‚ 151.421201813 ODeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚1720726 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚50327552 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 IBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚59 MADeviceStatsMonitor.on_thwùtrain_batch_end/allocation.large_pool.freed‚2020528 E6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚321 D=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚59 % _timestamp‚1.7866588028909633e+09 I=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚1839511 R?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚15037024496640 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 5368709120 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 H@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚665 MADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚1839511 G@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚77 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚209 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 123731968 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚321 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚1256408 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚59395584 BDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654601216 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 MADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚1843802 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚321 trainer/global_step‚549 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 122904576 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 5492441088 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 5368709120 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚321 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 4654601216 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 4552982528 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚41947136 ]JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚14905142804480 YFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚14905125699584 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 4552982528 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚41947136 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚262 F:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚3864332 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 G@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚59 _step‚43 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 122904576 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚20643840˜FŽˆž&À˜M E6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚321 D=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚59 % _timestamp‚1.7866588028909633e+09 I=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚1839511 R?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚15037024496640 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 5368709120 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 H@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚665 MADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚1839511 G@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚77 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚209 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 123731968 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚321 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚1256408 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚59395584 BDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654601216 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 MADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚1843802 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚321 trainer/global_step‚549 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 122904576 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 5492441088 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 5368709120 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚321 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 4654601216 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 4552982528 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚41947136 ]JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚14905142804480 YFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚14905125699584 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 4552982528 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚41947136 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚262 F:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚3864332 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 G@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚59 _step‚43 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 122904576 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚20643840 F:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚3860039 I@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚4874 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚41348060 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚262 B6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚3860039 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 122306518 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚429 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 I=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚2020528 QEDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚1843802 [HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚13151841394688 ^KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚13012286046208 `ODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 139534704640 \KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 139484377088 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚4293 >5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5262 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17104896 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 4517085184 [JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 131663796019 NGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚75 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 831258624 B9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚5262 TIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚464395 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 123731968 DDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚3864332 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521198 TCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 131898797056 EDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 NBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚1720803 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 R?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚14950984526679 ]JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚14819379118080 WFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 131898797056 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 843096576 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚4291 K>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59052032 [JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 131940744192 QEDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚2020530 WFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 131622447959 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚262 VCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚15037083548672 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 123731968 _runtime‚ 151.421201813 ODeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚1720726 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚50327552 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 IBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚59 MADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚2020528+e·Ø0Á+ _wandbruntime‚151  _runtime‚151Ë™lÂæ 0loss/FRt2/assignment_loss‚1.3707172231746656 /loss/FRt1/detection_loss‚0.4868751077405819 $loss/total_loss‚4.05515010904262 _runtime‚ 151.421693603 0loss/FRt1/assignment_loss‚1.5958508007359193 /loss/FRt2/detection_loss‚0.6017069773914533 epoch‚1 trainer/global_step‚549 $ _timestamp‚1.786658802891905e+09 _step‚44ò»4 pÃê _step‚44 0loss/FRt2/assignment_loss‚1.3707172231746656 /loss/FRt1/detection_loss‚0.4868751077405819 $loss/total_loss‚4.05515010904262 _runtime‚ 151.421693603 0loss/FRt1/assignment_loss‚1.5958508007359193 /loss/FRt2/detection_loss‚0.6017069773914533 epoch‚1 trainer/global_step‚549 $ _timestamp‚1.786658802891905e+09,ÖeÑ0Ä+ _wandbruntime‚163  _runtime‚163h7bÄž & lr-AdamW/pg1‚0.0014996185055696717 & lr-AdamW/pg2‚0.0014996185055696717 trainer/global_step‚599 % _timestamp‚1.7866588146544385e+09 _runtime‚ 163.184426445 _step‚45àÐv>ÈÆ % _timestamp‚1.7866588146544385e+09 _runtime‚ 163.184426445 _step‚45 & lr-AdamW/pg1‚0.0014996185055696717 & lr-AdamW/pg2‚0.0014996185055696717 trainer/global_step‚599-®Ëîj0Ç+ _wandbruntime‚163  _runtime‚163Ú"»r'ÈŠO =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚321 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 4552982528 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚41348012 YHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 142953685955 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 4654601216 XEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚16245478035311 M@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚59048960 ]JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚14293189496832 KDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚59 YFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚14293124813824 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 5368709120 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚41944064 D;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚5262 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚429 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 5492441088 F?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚59 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚41944064 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 122904576 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚223 TADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚16245419647939 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 122306518 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚262 trainer/global_step‚599 VEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 143254494208 L@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚4200238 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚20643840 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚1364451 ZIDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 143296438272 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17104896 IBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚74 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚59395584 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 % _timestamp‚1.7866588146553555e+09 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 PDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚1868821 G>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4874 [HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚16194920251392 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 123731968 HDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚16338174745600 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 K?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚2194625 OCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚2005611 JBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚665 SGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚2005611 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚209 _runtime‚ 163.188885724 ]LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 143296438272 `MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚14141607313408 YHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 143254494208 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚1364453 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚321 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 5368709120 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚262 @7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚5262 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 4654601216 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4287 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚44039168 PCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚58387372 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 F>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚4285 KBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚4874 VKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚504368 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚262 _LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚16194937356288 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 123731968 _step‚46 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 122904576 TADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚16338174745600 SGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚2194627 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 4628521198 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚4285 XEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚16338233794560 PCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚59048960 L@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚1868747 PIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚72 _LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚16102483001344 K?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚2001326 ^MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 151517500416 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 5492441088 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 5368709120 HDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 [HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚16102465961984 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 ]LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 142995033967 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚321 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚321 D8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚4195951 UBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚16338233794560 bQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 151561539584 BDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 [HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚16102465961984 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 ]LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 142995033967 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚321 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚321 D8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚4195951 UBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚16338233794560 bQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 151561539584 BDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4874 [HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚16194920251392 Ÿ¯í{ 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 123731968 HDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚16338174745600 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 K?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚2194625 OCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚2005611 JBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚665 SGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚2005611 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚209 _runtime‚ 163.188885724 ]LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 143296438272 `MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚14141607313408 YHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 143254494208 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚1364453 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚321 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 5368709120 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚262 @7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚5262 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 4654601216 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4287 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚44039168 PCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚58387372 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 F>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚4285 KBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚4874 VKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚504368 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚262.´¼(€0Ê+ _wandbruntime‚163  _runtime‚163úUV™&Ë“M UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17104896 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚429 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚41348060 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚321 F:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚4207084 MADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚2198178 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 B6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚4202791 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚1366658 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 S@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚16364787204096 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521198 D=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚59 NGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚69 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚20643840 ^KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚14164654686208 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚262 _step‚47 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚4293 YFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚16221242589184 YFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚16128651755520 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 123731968 trainer/global_step‚599 [HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚14316483273728 bODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚14164675330048 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚44036096 B5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5262 WFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 143485562880 J>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚4207084 F:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚4202791 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 WFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 143184258567 E8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 122306518 I=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚2198178 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚4291 J>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚1871768 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 IBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚59 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 4552982528 ZGDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚16221259694080 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 123731968 VCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚16364787204096 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 831258624 _runtime‚ 163.346083016 QEDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚2198180 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚321 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 R?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚16364728152064 \KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 151763907584 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 5368709120 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚321 G@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚59 WDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚14316418593792 `ODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 151807943680 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚262 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚4291 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚223 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 VCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚16271894401507 [JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 143225606627 D6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚321 NADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚59052032 ]JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚16221259694080 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 5492441088 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654601216 MADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚2198180 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚41947136 TIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚505181 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 5492441088 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 NADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚58387420 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚41947136 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 4552982528 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 4517085184 ODeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59052032 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 4654601216 XGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 143527510016 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17104896 QEDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚2008904 G@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚71 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚209 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚21ˆL&Ì—M WFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 143485562880 J>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚4207084 F:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚4202791 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 WFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 143184258567 E8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 122306518 I=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚2198178 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚4291 J>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚1871768 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 IBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚59 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 4552982528 ZGDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚16221259694080 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 123731968 VCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚16364787204096 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 831258624 _runtime‚ 163.346083016 QEDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚2198180 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚321 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 R?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚16364728152064 \KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 151763907584 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 5368709120 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚321 G@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚59 WDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚14316418593792 `ODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 151807943680 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚262 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚4291 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚223 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 VCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚16271894401507 [JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 143225606627 D6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚321 NADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚59052032 ]JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚16221259694080 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 5492441088 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654601216 MADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚2198180 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚41947136 TIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚505181 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 5492441088 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 NADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚58387420 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚41947136 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 4552982528 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 4517085184 ODeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59052032 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 4654601216 XGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 143527510016 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17104896 QEDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚2008904 G@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚71 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚209 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17104896 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚429 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚41348060 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚321 F:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚4207084 MADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚2198178 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 B6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚4202791 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚1366658 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 S@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚16364787204096 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521198 D=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚59 NGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚69 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚20643840 ^KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚14164654686208 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚262 _step‚47 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚4293 YFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚16221242589184 YFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚16128651755520 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 123731968 trainer/global_step‚599 [HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚14316483273728 bODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚14164675330048 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚44036096 B5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5262/Ί6E0Í+ _wandbruntime‚163  _runtime‚163„ÎV£mÎç % _timestamp‚1.7866588148167362e+09 _runtime‚ 163.346531413 0loss/FRt2/assignment_loss‚1.4600164974503655 %loss/total_loss‚4.395324715282757 epoch‚1 _step‚48 0loss/FRt1/assignment_loss‚1.7215298420753409 /loss/FRt1/detection_loss‚0.6291030410349434 .loss/FRt2/detection_loss‚0.584675334722107 trainer/global_step‚599µUüÎqÏë 0loss/FRt2/assignment_loss‚1.4600164974503655 %loss/total_loss‚4.395324715282757 epoch‚1 _step‚48 0loss/FRt1/assignment_loss‚1.7215298420753409 /loss/FRt1/detection_loss‚0.6291030410349434 .loss/FRt2/detection_loss‚0.584675334722107 trainer/global_step‚599 % _timestamp‚1.7866588148167362e+09 _runtime‚ 163.3465314130'3­¡Ð‚(:– €€ùÓù¹á gpu.0.gpu‚64.0 gpu.0.memory‚12, gpu.0.memoryAllocated‚7.0288848876953125) gpu.0.memoryAllocatedBytes‚ 6037766144 gpu.0.temp‚35.0 gpu.0.powerWatts‚139.189' gpu.0.enforcedPowerLimitWatts‚500.0) gpu.0.powerPercent‚27.837799999999994 gpu.0.smClock‚1410 gpu.0.memoryClock‚1593$ gpu.0.correctedMemoryErrors‚1955# gpu.0.uncorrectedMemoryErrors‚0ÔBmÑ‚(:â €€ùÓý£üâ disk.sda2.in‚0$ disk.nvme0n1.out‚481.39208984375 proc.memory.rssMB‚ 2285.78125 network.sent‚803973( disk./.usagePercent‚30.5721002415728" disk./.usageGB‚455.54638671875 disk.dm-1.in‚ 1224.7265625 disk.drbd2165.out‚0 disk.dm-0.in‚ 62.3671875& disk.nvme0c0n1.out‚481.39208984375 disk.sda1.in‚0 disk.sda3.in‚ 1224.7265625 disk.nvme0n1.in‚ 62.3671875) proc.memory.availableMB‚ 6005.84765625* proc.memory.percent‚13.951301574707031 cpu‚24.921325747608986 disk.dm-1.out‚2.96875 disk.drbd2165.in‚0! disk.dm-0.out‚481.39208984375 disk.nvme0c0n1.in‚ 62.3671875 disk.sda.out‚2.96875 disk.sda2.out‚0% memory_percent‚63.343214988708496 proc.cpu.threads‚17 network.recv‚408676 disk.sda3.out‚2.96875 disk.sda.in‚ 1224.7265625 disk.sda1.out‚0*æäT0Ò+ _wandbruntime‚175  _runtime‚175 J”˜ÄÓ¾ & lr-AdamW/pg1‚0.0014994027448336541 & lr-AdamW/pg2‚0.0014994027448336541 trainer/global_step‚649 % _timestamp‚1.7866588265752182e+09 _runtime‚ 175.105224376 _step‚49óaþüÈÔ _step‚49 & lr-AdamW/pg1‚0.0014994027448336541 & lr-AdamW/pg2‚0.0014994027448336541 trainer/global_step‚649 % _timestamp‚1.7866588265752182e+09 _runtime‚ 175.1052243761PÐú‘0Õ+ _wandbruntime‚175  _runtime‚175šú1µÖŠO SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 4552982528 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 _LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚17411772678144 ZIDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 154883204096 L@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚2019190 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚321 IBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚59 SGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚2170713 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚429 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 4517085184 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 K?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚2372275 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 ]LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 154556844575 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚209 _runtime‚ 175.109900702 SGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚2372277 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚262 F?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚59 HDeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4287 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚59395584 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 123731968 M@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚59048960 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 122904576 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 YHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 154515496563 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 5368709120 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 JBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚665 IBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚65 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 122306518 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 D8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚4538703 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 5492441088 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 5492441088 [HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚17511037140992 ]JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚15457849699840 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 843096576 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚20643840 `MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚15293975953408 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 PCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚58387372 F>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 OCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚2170713 YHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 154841260032 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚39844864 BDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚17665878401024 UBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚17665937449984 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 4552982528 OCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚2372275 HBDeviceStatsMonitor.on_train_batch_start/oveüè`žÛ rsize_allocations.freed‚0 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚321 OCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚2166428 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 5492441088 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 5368709120 _step‚50 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 4654601216 \IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚17511054245888 TADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚17665878401024 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17104896 UHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚60488704 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 PCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚59048960 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚41944064 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 PIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚63 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚262 OCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚2372277 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 122904576 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚1474701 TADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚17566271135347 KDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚59 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚223 YFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚15457789211136 XEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚17566329522719 K?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚2166428 D;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚5262 HDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4874 ]LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 154883204096 trainer/global_step‚649 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17104896 PDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚2019255sƒÒ”'׎O \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚20643840 `MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚15293975953408 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 PCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚58387372 F>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 OCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚2170713 YHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 154841260032 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚39844864 BDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚17665878401024 UBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚17665937449984 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 4552982528 OCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚2372275 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚321 OCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚2166428 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 5492441088 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 5368709120 _step‚50 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 4654601216 \IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚17511054245888 TADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚17665878401024 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17104896 UHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚60488704 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 PCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚59048960 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚41944064 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 PIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚63 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚262 OCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚2372277 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 122904576 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚1474701 TADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚17566271135347 KDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚59 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚223 YFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚15457789211136 XEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚17566329522719 K?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚2166428 D;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚5262 HDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4874 ]LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 154883204096 trainer/global_step‚649 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17104896 PDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚2019255 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 4552982528 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 _LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚17411772678144 ZIDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 154883204096 L@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚2019190 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚321 IBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚59 SGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚2170713 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚429 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 4517085184 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 K?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚2372275 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 ]LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 154556844575 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚209 _runtime‚ 175.109900702 SGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚2372277 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚262 F?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚59 HDeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4287 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚59395584 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 123731968 M@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚59048960 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 122904576 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 YHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 154515496563 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 5368709120 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 JBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚665 IBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚65 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 122306518 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 D8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚4538703 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 5492441088 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 5492441088 [HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚17511037140992 ]JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚15457849699840 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 843096576 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚02J§<{0Ø+ _wandbruntime‚175  _runtime‚175YVÚùš&Ù”M MADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚2375828 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚41938944 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 123731968 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚262 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚59395584 trainer/global_step‚649 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 4552982528 I@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚4874 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚262 I=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚2169715 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 5368709120 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 \KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 164059664896 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 G@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚59 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚223 E8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚41348060 [JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 155114275840 MADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚2169715 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚321 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 122904576 H@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚665 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 BDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚4293 EDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚2022183 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚209 ^KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚15317023326208 D=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚59 S@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚17692490859520 QEDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚2375830 NBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚2022256 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 122306518 F:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚4545543 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚1476908 YFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚17537359478784 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 4654601216 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚41947136 VCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚17692490859520 B6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚4545543 MADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚2375830 WDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚15481082991104 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 5492441088 I=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚2375828 ODeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521198 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 4517085184 K>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59052032 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚1476906 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚321 % _timestamp‚1.7866588267372532e+09 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚429 ]JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚17537376583680 bODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚15317043970048 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 IBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚59 F:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚4549836 DDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚4549836 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚321 ZGDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚17537376583680 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 PEDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚545277 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 VCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚17592745888915 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 123731968 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17104896 TCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 155072328704 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654601216 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚4291 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 4552982528 [HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚15481145573888 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 831258624 YFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚17437941432320 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚4291 G@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚73 >5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5262 TIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚545348 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 122904576 SFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚62582784 `ODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 164101603840 NADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚59052032 WFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 155072328704 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚2064384096C¤%Ú˜M J>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚2022183 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚209 ^KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚15317023326208 D=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚59 S@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚17692490859520 QEDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚2375830 NBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚2022256 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 122306518 F:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚4545543 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚1476908 YFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚17537359478784 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 4654601216 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚41947136 VCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚17692490859520 B6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚4545543 MADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚2375830 WDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚15481082991104 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 5492441088 I=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚2375828 ODeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521198 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 4517085184 K>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59052032 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚1476906 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚321 % _timestamp‚1.7866588267372532e+09 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚429 ]JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚17537376583680 bODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚15317043970048 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 IBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚59 F:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚4549836 DDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚4549836 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚321 ZGDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚17537376583680 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 PEDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚545277 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 VCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚17592745888915 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 123731968 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17104896 TCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 155072328704 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654601216 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚4291 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 4552982528 [HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚15481145573888 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 831258624 YFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚17437941432320 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚4291 G@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚73 >5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5262 TIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚545348 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 122904576 SFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚62582784 `ODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 164101603840 NADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚59052032 WFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 155072328704 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚20643840 MADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚2375828 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚41938944 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 123731968 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚262 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚59395584 trainer/global_step‚649 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 4552982528 I@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚4874 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚262 I=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚2169715 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 5368709120 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 \KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 164059664896 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 G@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚59 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚223 E8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚41348060 [JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 155114275840 MADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚2169715 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚321 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 122904576 H@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚665 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 BDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚4293 EDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4874 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚20643840 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 YHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 166077307171 F?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚59 D8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚4881455 ]LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 166469969920 SGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚2335815 VKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚585354 trainer/global_step‚699 OCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚2549925 dQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚16446365237248 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 XEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚18887181010127 K?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚2331530 RGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚585291 _runtime‚ 187.02989237 YHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 166428025856 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚46136320 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚321 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 5492441088 \IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚18827171135488 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚209 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 4654601216 bQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 176134058496 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 5368709120 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚321 [HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚18721045315584 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚4285 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚223 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 831258624 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚262 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚41944064 D;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚5262 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 4628521198 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 122306518 HDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚18993582056448 L@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚2170242 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17104896 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 TADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚18993582056448 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚429 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 123731968 OCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚2331530 XEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚18993641105408 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 4654601216 K?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚2549925 ZIDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 166469969920 UHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚66780160 `MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚16446344593408 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 BDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 YFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚16622432515584 PCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚59048960 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 4552982528 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4287 _step‚54 IBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚65 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 PIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚63 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚321 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚41944064 @7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚5262 XEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚18827154030592 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 OCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚2335815 HDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 _LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚18721062354944 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 123731968 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚321 % _timestamp‚1.7866588384964411e+09 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 122904576 [HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚18827154030592 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 TADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚18887122622755 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 IBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚59 C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚4287 OCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚2549927 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0ÛwY´“'åO JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚262 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚41944064 D;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚5262 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 4628521198 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 122306518 HDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚18993582056448 L@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚2170242 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17104896 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 TADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚18993582056448 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚429 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 123731968 OCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚2331530 XEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚18993641105408 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 4654601216 K?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚2549925 ZIDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 166469969920 UHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚66780160 `MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚16446344593408 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 BDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 YFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚16622432515584 PCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚59048960 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 4552982528 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4287 _step‚54 IBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚65 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 PIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚63 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚321 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚41944064 @7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚5262 XEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚18827154030592 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 OCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚2335815 HDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 _LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚18721062354944 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 123731968 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚321 % _timestamp‚1.7866588384964411e+09 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 122904576 [HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚18827154030592 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 TADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚18887122622755 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 IBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚59 C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚4287 OCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚2549927 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚59395584 PCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚58387372 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚262 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17104896 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚4285 G>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4874 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚20643840 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 YHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 166077307171 F?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚59 D8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚4881455 ]LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 166469969920 SGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚2335815 VKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚585354 trainer/global_step‚699 OCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚2549925 dQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚16446365237248 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 XEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚18887181010127 K?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚2331530 RGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚585291 _runtime‚ 187.02989237 YHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 166428025856 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚46136320 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚321 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 5492441088 \IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚18827171135488 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚209 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 4654601216 bQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 176134058496 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 5368709120 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚321 [HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚18721045315584 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚4285 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚223 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 8312586246ó=¢(0æ+ _wandbruntime‚187  _runtime‚187fÄ»øE%ç”M S@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚19020194514944 QEDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚2553480 NBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚2173317 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 YFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚18747231109120 WFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 166307879783 bODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚16469412610048 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521198 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 122904576 NGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚54 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 843096576 IBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚59 B6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚4888295 R?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚19020135462912 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 [JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 166701041664 EDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59052032 VCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚18853476368384 WDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚16645724395008 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 5492441088 F:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚4892588 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17104896 J>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚2173261 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 831258624 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚41947136 MADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚2334817 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 123731968 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚429 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚1587158 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚262 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚41947136 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 4517085184 _runtime‚ 187.186361468 MADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚2553480 ODeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚4892588 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚4291 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚321 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 4552982528 `ODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 176380659200 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 5368709120 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 G@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚56 [HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚16645793269248 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 I=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚2334817 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17104896 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚209 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 R?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚18913538988903 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 122904576 F:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚4888295 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 123731968 D=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚59 >5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5262 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚20643840 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 VCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚18913597376323 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚262 _step‚55 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚4291 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654601216 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚1587156 ^KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚16469391966208 \KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 176332428800 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 123731968 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 [JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 166349227843 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚321 % _timestamp‚1.7866588386561062e+09 VCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚19020194514944 B9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚5262 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 5492441088 ]JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚18853493473280 SFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚68874240 E6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚321 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚4293 ZGDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚18853493473280 I@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚4874 TCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 166659094528 MADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚2553478 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 5492441088 MADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚2339108 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 4552982528 NADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚59052032 TIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚58¤òfÒU6159 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚262 trainer/global_step‚699 I=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚2553478 PEDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚586105 NADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚58387420ùAî!ž&è˜M TCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 166659094528 MADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚2553478 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 5492441088 MADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚2339108 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 4552982528 NADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚59052032 TIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚586159 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚262 trainer/global_step‚699 I=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚2553478 PEDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚586105 NADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚58387420 S@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚19020194514944 QEDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚2553480 NBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚2173317 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 YFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚18747231109120 WFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 166307879783 bODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚16469412610048 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521198 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 122904576 NGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚54 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 843096576 IBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚59 B6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚4888295 R?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚19020135462912 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 [JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 166701041664 EDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59052032 VCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚18853476368384 WDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚16645724395008 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 5492441088 F:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚4892588 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17104896 J>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚2173261 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 831258624 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚41947136 MADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚2334817 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 123731968 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚429 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚1587158 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚262 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚41947136 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 4517085184 _runtime‚ 187.186361468 MADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚2553480 ODeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚4892588 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚4291 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚321 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 4552982528 `ODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 176380659200 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 5368709120 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 G@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚56 [HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚16645793269248 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 I=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚2334817 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17104896 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚209 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 R?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚18913538988903 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 122904576 F:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚4888295 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 123731968 D=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚59 >5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5262 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚20643840 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 VCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚18913597376323 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚262 _step‚55 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚4291 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654601216 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚1587156 ^KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚16469391966208 \KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 176332428800 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 123731968 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 [JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 166349227843 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚321 % _timestamp‚1.7866588386561062e+09 VCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚19020194514944 B9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚5262 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 5492441088 ]JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚18853493473280 SFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚68874240 E6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚321 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚4293 ZGDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚18853493473280 I@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚48747#\–0é+ _wandbruntime‚187  _runtime‚187c¨ mêç _step‚56 /loss/FRt1/assignment_loss‚1.008365992298606 /loss/FRt2/detection_loss‚0.5147690356781628 %loss/total_loss‚3.072860928974745 epoch‚1 trainer/global_step‚699 0loss/FRt2/assignment_loss‚0.9244975570468958 /loss/FRt1/detection_loss‚0.6252283439510805 % _timestamp‚1.7866588386570797e+09 _runtime‚ 187.186877543+²qëë /loss/FRt1/assignment_loss‚1.008365992298606 /loss/FRt2/detection_loss‚0.5147690356781628 %loss/total_loss‚3.072860928974745 epoch‚1 trainer/global_step‚699 0loss/FRt2/assignment_loss‚0.9244975570468958 /loss/FRt1/detection_loss‚0.6252283439510805 % _timestamp‚1.7866588386570797e+09 _runtime‚ 187.186877543 _step‚568ÇàÎe–ì‚(:‹ ž€ùÓÜ„öà gpu.0.gpu‚61.0 gpu.0.memory‚11, gpu.0.memoryAllocated‚7.0288848876953125) gpu.0.memoryAllocatedBytes‚ 6037766144 gpu.0.temp‚36.0 gpu.0.powerWatts‚224.109' gpu.0.enforcedPowerLimitWatts‚500.0 gpu.0.powerPercent‚44.8218 gpu.0.smClock‚1410 gpu.0.memoryClock‚1593$ gpu.0.correctedMemoryErrors‚1955# gpu.0.uncorrectedMemoryErrors‚0*ž%psí‚(:è ž€ùÓ¨îÛâ disk.sda.out‚3.453125 disk.dm-1.in‚ 1224.7265625 disk.drbd2165.out‚0! disk.dm-0.out‚823.63232421875$ memory_percent‚63.34497928619385* proc.memory.percent‚13.951301574707031 disk.sda1.in‚0) proc.memory.availableMB‚ 6005.55859375 cpu‚25.209479974013366 proc.cpu.threads‚17 network.recv‚462375$ disk./.usageGB‚456.0470199584961 disk.sda3.in‚ 1224.7265625 disk.sda3.out‚3.453125 disk.sda2.out‚0 disk.dm-1.out‚3.453125 disk.dm-0.in‚ 62.3671875 proc.memory.rssMB‚ 2285.78125 network.sent‚963506* disk./.usagePercent‚30.605698158351437 disk.nvme0c0n1.in‚ 62.3671875 disk.sda1.out‚0 disk.sda2.in‚0 disk.nvme0n1.in‚ 62.3671875$ disk.nvme0n1.out‚823.63232421875 disk.drbd2165.in‚0& disk.nvme0c0n1.out‚823.63232421875 disk.sda.in‚ 1224.7265625ëÔz0î+ _wandbruntime‚198  _runtime‚198£•Äï¾ & lr-AdamW/pg2‚0.0014988268420972211 trainer/global_step‚749 % _timestamp‚1.7866588504597404e+09 _runtime‚ 198.989843665 _step‚57 & lr-AdamW/pg1‚0.0014988268420972211V*SÒÈð & lr-AdamW/pg1‚0.0014988268420972211 & lr-AdamW/pg2‚0.0014988268420972211 trainer/global_step‚749 % _timestamp‚1.7866588504597404e+09 _runtime‚ 198.989843665 _step‚579Ä#Åc0ñ+ _wandbruntime‚198  _runtime‚198‡i-'òŠO @7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚5262 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17104896 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚1695203 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 UHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚64683008 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 YFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚17787083445248 F?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚59 % _timestamp‚1.7866588504607215e+09 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚44039168 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚321 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 123731968 C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚4287 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 4552982528 D;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚5262 PCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚58387372 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 TADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚20321285711872 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚4285 ZIDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 178056735744 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚41944064 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 5368709120 KDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚59 ]LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 178056735744 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 122306518 trainer/global_step‚749 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4287 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 _LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚20030352031744 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚321 UBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚20321344760832 JBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚665 L@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚2321437 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 _runtime‚ 198.993889497 YHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 178014791680 HDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4874 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 IBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚65 PIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚63 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚209 XEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚20208032497535 YHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 177639117779 K?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚2496632 KBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚4874 RGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚626236 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 843096576 SGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚2727577 `MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚17598713233408 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚41348012 OCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚2727577 PDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚2321502 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 4628521198 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 5368709120 \IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚20143288025088 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 4654601216 L@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚5228494 IBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚59 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚41944064 _LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚20143288025088 ]JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚17787148128256 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 831258624 BDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 4654601216 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚20643840 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 [HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚20030334992384 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 123731968 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17104896 XEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚20143270920192 XEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚20321344760832 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 122904576 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 5368709120 HDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚20321285711872 ^MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 188370211840 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚4285 _step‚58 F>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 M@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚59048960 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 5492441088 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚262 [HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚20143270920192 ]LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 177680465791m,t¶&óŽO SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 5492441088 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚262 [HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚20143270920192 ]LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 177680465791 @7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚5262 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17104896 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚1695203 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 UHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚64683008 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 YFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚17787083445248 F?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚59 % _timestamp‚1.7866588504607215e+09 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚44039168 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚321 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 123731968 C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚4287 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 4552982528 D;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚5262 PCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚58387372 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 TADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚20321285711872 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚4285 ZIDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 178056735744 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚41944064 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 5368709120 KDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚59 ]LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 178056735744 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 122306518 trainer/global_step‚749 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4287 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 _LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚20030352031744 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚321 UBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚20321344760832 JBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚665 L@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚2321437 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 _runtime‚ 198.993889497 YHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 178014791680 HDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4874 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 IBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚65 PIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚63 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚209 XEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚20208032497535 YHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 177639117779 K?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚2496632 KBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚4874 RGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚626236 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 843096576 SGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚2727577 `MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚17598713233408 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚41348012 OCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚2727577 PDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚2321502 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 4628521198 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 5368709120 \IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚20143288025088 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 4654601216 L@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚5228494 IBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚59 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚41944064 _LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚20143288025088 ]JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚17787148128256 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 831258624 BDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 4654601216 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚20643840 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 [HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚20030334992384 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 123731968 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17104896 XEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚20143270920192 XEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚20321344760832 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 122904576 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 5368709120 HDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚20321285711872 ^MDeviceStatsMonitor.on_tr-Ðy`uain_batch_start/inactive_split_bytes.small_pool.freed‚ 188370211840 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚4285 _step‚58 F>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 M@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚59048960:º³ÌÔ0ô+ _wandbruntime‚199  _runtime‚199A€š&õ”M ]JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚20169610362880 NBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚2324518 % _timestamp‚1.7866588506224399e+09 G@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚63 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚1697408 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚209 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 _runtime‚ 199.152878946 DDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚5235340 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 R?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚20234390476311 G@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚59 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 VCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚20347898170368 [HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚17810441053184 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 4517085184 ZGDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚20169610362880 R?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚20347839118336 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚429 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 NADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚58387420 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 \KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 188615767040 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 5492441088 VCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚20169593257984 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚321 B6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚321 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚262 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17104896 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚20643840 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 ODeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654601216 E8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 122904576 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 122306518 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 5368709120 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚262 MADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚2499919 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 123731968 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 4552982528 WFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 178245860352 WDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚17810376373248 `ODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 188659803136 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 5368709120 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚262 IBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚59 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 5492441088 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚321 VCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚20234448863731 MADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚2731128 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 I=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚2731128 I@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚4874 SFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚64679936 MADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚2504210 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 122904576 H@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚665 TIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚627110 bODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚17621781250048 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 trainer/global_step‚749 I=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚2499919 EDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚2324455 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 5368709120 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 123731968 MADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚2731130 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521198 ]JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚20056537825280 [JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 177911038451 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 >5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5262 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚4293 B6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚5231047 NADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚59052032 QEDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚2504210 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚41348060 K>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59052032 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17104896 PEDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚627049 ^KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚17621760606208 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 831258624 QEDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚2731130 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚59395584 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚41947136G@‰$ž&ö˜M VCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚20234448863731 MADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚2731128 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 I=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚2731128 I@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚4874 SFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚64679936 MADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚2504210 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 122904576 H@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚665 TIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚627110 bODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚17621781250048 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 trainer/global_step‚749 I=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚2499919 EDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚2324455 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 5368709120 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 123731968 MADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚2731130 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521198 ]JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚20056537825280 [JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 177911038451 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 >5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5262 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚4293 B6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚5231047 NADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚59052032 QEDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚2504210 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚41348060 K>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59052032 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17104896 PEDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚627049 ^KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚17621760606208 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 831258624 QEDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚2731130 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚59395584 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚41947136 ]JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚20169610362880 NBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚2324518 % _timestamp‚1.7866588506224399e+09 G@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚63 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚1697408 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚209 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 _runtime‚ 199.152878946 DDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚5235340 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 R?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚20234390476311 G@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚59 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 VCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚20347898170368 [HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚17810441053184 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 4517085184 ZGDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚20169610362880 R?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚20347839118336 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚429 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 NADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚58387420 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 \KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 188615767040 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 5492441088 VCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚20169593257984 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚321 B6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚321 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚262 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17104896 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚20643840 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 ODeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654601216 E8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 122904576 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 122306518 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 5368709120 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚262 MADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚2499919 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 123731968 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 4552982528 WFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 178245860352 WDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚17810376373248 `ODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 188659803136 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 5368709120 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚262 IBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚59 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 5492441088 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚321;®Go0÷+ _wandbruntime‚199  _runtime‚199]þä"møç /loss/FRt1/detection_loss‚0.6018255624171007 _runtime‚ 199.153416462 /loss/FRt1/assignment_loss‚1.258492855062823 /loss/FRt2/detection_loss‚0.5580399008869584 &loss/total_loss‚3.4285668636190874 epoch‚1 trainer/global_step‚749 $ _timestamp‚1.786658850623576e+09 _step‚60 0loss/FRt2/assignment_loss‚1.0102085452522056½‘áÜqùë _step‚60 0loss/FRt2/assignment_loss‚1.0102085452522056 /loss/FRt1/detection_loss‚0.6018255624171007 _runtime‚ 199.153416462 /loss/FRt1/assignment_loss‚1.258492855062823 /loss/FRt2/detection_loss‚0.5580399008869584 &loss/total_loss‚3.4285668636190874 epoch‚1 trainer/global_step‚749 $ _timestamp‚1.786658850623576e+09< ¶Í•ú‚(:Š ­€ùÓ¦ûþà gpu.0.gpu‚46.0 gpu.0.memory‚10, gpu.0.memoryAllocated‚7.0288848876953125) gpu.0.memoryAllocatedBytes‚ 6037766144 gpu.0.temp‚37.0 gpu.0.powerWatts‚157.935' gpu.0.enforcedPowerLimitWatts‚500.0 gpu.0.powerPercent‚31.587 gpu.0.smClock‚1410 gpu.0.memoryClock‚1593$ gpu.0.correctedMemoryErrors‚1955# gpu.0.uncorrectedMemoryErrors‚0Híš{û‚(:ð ­€ùÓûˤã disk.drbd2165.out‚0 disk.sda3.out‚3.609375 disk.sda.in‚ 1224.7265625 disk.sda.out‚3.609375 proc.memory.rssMB‚ 2285.78125 network.recv‚493192 disk.sda1.out‚0 disk.sda2.in‚0$ disk.nvme0n1.out‚830.02685546875 disk.sda3.in‚ 1224.7265625* proc.memory.percent‚13.951301574707031 cpu‚25.327168778040043 proc.cpu.threads‚17 network.sent‚ 1.036349e+06 disk.sda1.in‚0 disk.sda2.out‚0 disk.dm-1.in‚ 1224.7265625! disk.dm-0.out‚830.02685546875 disk.nvme0c0n1.in‚ 62.3671875) proc.memory.availableMB‚ 6005.08984375% disk./.usageGB‚456.04676818847656 disk.dm-1.out‚3.609375 disk.dm-0.in‚ 62.3671875& disk.nvme0c0n1.out‚830.02685546875% memory_percent‚63.347840309143066* disk./.usagePercent‚30.605681261854173 disk.nvme0n1.in‚ 62.3671875 disk.drbd2165.in‚0§Æ0ü+ _wandbruntime‚211  _runtime‚211Û:rðÄý¾ & lr-AdamW/pg1‚0.0014984667370859623 & lr-AdamW/pg2‚0.0014984667370859623 trainer/global_step‚799 % _timestamp‚1.7866588628117285e+09 _runtime‚ 211.341779415 _step‚61Ô½ŽÈþ & lr-AdamW/pg1‚0.0014984667370859623 & lr-AdamW/pg2‚0.0014984667370859623 trainer/global_step‚799 % _timestamp‚1.7866588628117285e+09 _runtime‚ 211.341779415 _step‚61= M¹0ÿ+ _wandbruntime‚211  _runtime‚211¨ùÌC·&€‰O G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚429 PIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚82 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚20643840 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 XEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚21528883984943 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 5492441088 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚321 KDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚59 _runtime‚ 211.345886466 XEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚21649048416256 OCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚2905227 ZIDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 189643501568 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 4517085184 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 123731968 _LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚21459404914688 BDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 G>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4874 L@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚5571246 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 4628521198 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 4654601216 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17104896 JBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚665 UHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚66780160 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 843096576 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚262 XEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚21459387809792 SGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚2905227 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚1805453 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 122306518 OCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚2905225 PDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚2473069 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 OCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚2661734 M@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚59048960 [HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚21459387809792 ]LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 189643501568 YHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 189601557504 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚46136320 PCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚58387372 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 5368709120 D8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚5566959 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 Q>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚21648989367296 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 122904576 KBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚4874 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚41348012 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚262 HDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 SGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚2666019 VKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚667616 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 L@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚2472985 _LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚21339641708544 [HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚21339624669184 YHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 189200928387 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 123731968 OCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚2666019 PCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚59048960 TADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚21648989367296 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚41944064 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 5492441088 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4287 ^MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 200651462144 C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚4287 \IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚214594h†ÂVØ04914688 HDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 G>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4874 L@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚5571246 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 4628521198 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 4654601216 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17104896 JBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚665 UHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚66780160 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 843096576 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚262 XEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚21459387809792 SGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚2905227 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚1805453 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 122306518 OCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚2905225 PDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚2473069 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 OCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚2661734 M@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚59048960 [HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚21459387809792 ]LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 189643501568 YHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 189601557504 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚46136320 PCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚58387372 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 5368709120 D8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚5566959 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 Q>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚21648989367296 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 122904576 KBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚4874 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚41348012 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚262 HDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 SGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚2666019 VKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚667616 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 L@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚2472985 _LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚21339641708544 [HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚21339624669184 YHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 189200928387 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 123731968 OCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚2666019 PCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚59048960 TADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚21648989367296 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚41944064 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 5492441088 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4287 ^MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 200651462144 C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚4287 \IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚21459404914688>Áà~0‚+ _wandbruntime‚211  _runtime‚211ñÄÙ ™&ƒ“M JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 123731968 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚4293 R?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚21675542773760 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17104896 TIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚668421 PEDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚668344 B6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚321 IBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚59 YFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚21485710147584 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚4291 G@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚79 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 5368709120 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚262 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚50327552 NADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚58387420 WFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 189431500999 D8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 F:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚5578092 QEDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚2908780 I@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚4874 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 831258624 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 123731968 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚4291 S@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚21675601825792 `ODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 200945575424 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚1807658 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 WDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚18975024494080 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 4552982528 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚41947136 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚262 I=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚2908778 MADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚2908778 bODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚18774149890048 ^KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚18774129246208 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521198 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 4552982528 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 5368709120 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚429 NGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚77 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚321 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 122306518 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 D=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚59 [JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 189874573312 B6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚5573799 >5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5262 MADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚2669312 XGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 189874573312 TCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 189832626176 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 ZGDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚21485727252480 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚59395584 VCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚21555300351139 _runtime‚ 211.503153966 F:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚5573799 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚1807656 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 EDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654601216 SFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚70971392 \KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 200895247872 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 $ _timestamp‚1.786658862972922e+09 ]JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚21485727252480 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚223 _step‚63 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17104896 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚41947136 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 YFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚21365810462720 [JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 189472849059 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚41348060 NADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚59052032 WFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 189832626176 [HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚18975095465472 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚321 R?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚21555241963719 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 123731968 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚262 MADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚2908780 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 122904576 J>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚5578092 H@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚665 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 J>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚2476000 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 5368709120 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚321 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 K>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59052032 O6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚321 IBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚59 YFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚21485710147584 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚4291 G@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚79 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 5368709120 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚262 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚50327552 NADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚58387420 WFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 189431500999 D8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 F:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚5578092 QEDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚2908780 I@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚4874 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 831258624 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 123731968 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚4291 S@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚21675601825792 `ODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 200945575424 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚1807658 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 WDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚18975024494080 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 4552982528 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚41947136 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚262 I=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚2908778 MADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚2908778 bODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚18774149890048 ^KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚18774129246208 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521198 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 4552982528 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 5368709120 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚429 NGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚77 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚321 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 122306518 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 D=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚59 [JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 189874573312 B6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚5573799 >5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5262 MADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚2669312 XGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 189874573312 TCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 189832626176 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 ZGDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚21485727252480 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚59395584 VCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚21555300351139 _runtime‚ 211.503153966 F:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚5573799 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚1807656 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 EDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654601216 SFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚70971392 \KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 200895247872 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 $ _timestamp‚1.786658862972922e+09 ]JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚21485727252480 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚223 _step‚63 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17104896 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚41947136 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 YFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚21365810462720 [JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 189472849059 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚41348060 NADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚59052032 WFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 189832626176 [HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚18975095465472 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚321 R?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚21555241963719 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 123731968 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚262 MADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚2908780 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 122904576 J>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚5578092 H@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚665 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 J>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚2476000 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 5368709120 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚321 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 K>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59052032 ODeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚223 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚321 VEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 205256047104 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚1956315 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚209 YFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚20419755696640 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 bQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 217083108352 HDeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4288 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 5368709120 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚262 _LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚23166459019264 YHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 205256047104 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 PCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚58387376 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚321 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚262 % _timestamp‚1.7866588812620153e+09 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚4286 ^MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 217034875392 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 5492441088 L@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚2674446 TADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚23234642548917 F?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚59 TADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚23371697961472 OCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚2853309 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚41944576 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 122905088 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚4286 L@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚5996792 K?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚3143481 UBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚23371757010944 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 SGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚2853309 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚321 trainer/global_step‚849 VKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚718204 PIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚73 `MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚20202720821248 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 4517085184 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚262 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 4654601728 @7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚5263 JBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚665 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 PCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚59049472 G>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4875 BDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚23371697961472 ]JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚20419824573440 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 K?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚2849023 D;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚5263 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 _runtime‚ 229.795968245 OCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚3143483 \IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚23166459019264 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚171048967ëý”'’ŽO OCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚2849023 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 M@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚59049472 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚41944576 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 4552982528 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4288 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 5368709120 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚262 _LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚23166459019264 YHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 205256047104 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 PCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚58387376 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚321 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚262 % _timestamp‚1.7866588812620153e+09 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚4286 ^MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 217034875392 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 5492441088 L@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚2674446 TADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚23234642548917 F?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚59 TADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚23371697961472 OCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚2853309 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚41944576 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 122905088 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚4286 L@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚5996792 K?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚3143481 UBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚23371757010944 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 SGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚2853309 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚321 trainer/global_step‚849 VKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚718204 PIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚73 `MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚20202720821248 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 4517085184 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚262 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 4654601728 @7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚5263 JBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚665 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 PCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚59049472 G>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4875 BDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚23371697961472 ]JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚20419824573440 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 K?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚2849023 D;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚5263 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 _runtime‚ 229.795968245 OCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚3143483 \IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚23166459019264 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17104896 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 4628521202 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 123731968 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 4552982528 ZIDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 205297991680 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 831258624 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 4654601728 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚59395584 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 123731968 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 IBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚75 YHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 204828711093 RGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚718131 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚20643840 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 XEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚23234700936293 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚41348016 KDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚59 C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚4288 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 122905088 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17104896 HDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚223 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚321 VEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 205256047104 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚1956315 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚209 YFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚20419755696640 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 bQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 217083108352 HDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 XGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 205529063424 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17104896 QEDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚2856602 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚321 I@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚4875 EDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59052544 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚41947648 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654601728 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 5368709120 NADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚59052544 J>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚6003638 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 NADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚58387424 PEDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚718958 VCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚23261117302489 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521202 S@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚23398310420480 E5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5263 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 4552982528 H@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚665 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚209 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚321 trainer/global_step‚849 B6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚5999344 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17104896 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 I=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚3147034 bODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚20225788837888 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚321 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 5368709120 DDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚2677478 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 4517085184 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 123731968 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚321 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚262 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 4654601728 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 D=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚59 F:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚5999344 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 G@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚59 I=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚2852310 [JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 205529063424 MADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚2852310 NGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚72 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 R?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚23261058915065 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 123731968 [HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚20443115401216 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚41938432 NBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚2677552 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚59395584 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚262 O8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 BDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 MADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚3147036 MADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚3147034 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0  ˜‚¯ED>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0ö¢ Ôž&•˜M TIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚719030 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 5492441088 _runtime‚ 229.963818469 ^KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚20225768194048 >5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5263 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 4552982528 H@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚665 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚209 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚321 trainer/global_step‚849 B6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚5999344 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17104896 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 I=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚3147034 bODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚20225788837888 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚321 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 5368709120 DDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚2677478 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 4517085184 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 123731968 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚321 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚262 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 4654601728 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 D=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚59 F:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚5999344 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 G@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚59 I=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚2852310 [JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 205529063424 MADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚2852310 NGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚72 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 R?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚23261058915065 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 123731968 [HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚20443115401216 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚41938432 NBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚2677552 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚59395584 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚262 O8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 BDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 MADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚3147036 MADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚3147034 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 R?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚23398251367936 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚4292 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚1958520 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚20643840 IBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚59 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 [JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 205100631769 VCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚23398310420480 QEDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚3147036 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚4292 \KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 217284624896 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 XGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 205529063424 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17104896 QEDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚2856602 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚321 I@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚4875 EDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59052544 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚41947648 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654601728 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 5368709120 NADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚59052544 J>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚6003638 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 NADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚58387424 PEDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚718958 VCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚23261117302489 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521202 S@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚23398310420480 EDeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4288 dQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚21355110105088 PCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚58387376 K?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚3321131 Q>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚24699401616896 D;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚5263 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚262 ]LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 216431869717 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 4552982528 OCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚3014125 L@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚2826085 bQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 229353316352 XEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚24555552423701 XEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚24699460666368 TADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚24699401616896 ]LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 216884757504 IBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚76 _step‚71 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 122905088 UHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚73071104 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚52427264 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 123731968 UBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚24699460666368 HDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4875 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 BDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚2066567 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 5492441088 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 5368709120 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚321 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 122905088 OCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚3321131 JBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚665 KBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚4875 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚2066565 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 4628521202 F?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚59 _runtime‚ 242.384009146 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 5492441088 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 123731968 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 IBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚59 K?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚3014125 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚41944576 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 4552982528 YHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 216842812928 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚262 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17104896 _LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚24339120553984 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 5492441088 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚262 [HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚24339103514624 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 4654601728 XEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚24482558803968 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚209 ZIDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 216884757504 @7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚5263 PDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚2826161 ]JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚21584463421440 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚20643840 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 KDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚59 F>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 5368709120°½-v+' ŽO UHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚73071104 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚52427264 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 123731968 UBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚24699460666368 HDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4875 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 BDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚2066567 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 5492441088 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 5368709120 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚321 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 122905088 OCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚3321131 JBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚665 KBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚4875 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚2066565 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 4628521202 F?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚59 _runtime‚ 242.384009146 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 5492441088 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 123731968 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 IBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚59 K?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚3014125 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚41944576 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 4552982528 YHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 216842812928 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚262 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17104896 _LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚24339120553984 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 5492441088 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚262 [HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚24339103514624 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 4654601728 XEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚24482558803968 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚209 ZIDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 216884757504 @7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚5263 PDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚2826161 ]JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚21584463421440 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚20643840 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 KDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚59 F>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 5368709120 VEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 216842812928 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4288 dQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚21355110105088 PCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚58387376 K?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚3321131 Q>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚24699401616896 D;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚5263 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚262 ]LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 216431869717 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 4552982528 OCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚3014125 L@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚2826085 bQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 229353316352 XEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚24555552423701 XEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚24699460666368 TADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚24699401616896 ]LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 216884757504 IBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚#·–`i76 _step‚71 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 122905088G \…0¡+ _wandbruntime‚242  _runtime‚242ﺮðš&¢”M UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚41947648 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚4292 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚2068770 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 843096576 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚321 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 ZGDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚24508898246656 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚20643840 \KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 229548377600 NADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚58387424 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 122306522 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚321 VCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚24508881141760 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚41348064 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 831258624 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 MADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚3324686 G@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚73 R?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚24581910402473 _step‚72 TIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚760391 D=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚59 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 4654601728 B9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚5263 MADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚3021704 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654601728 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚321 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17104896 E5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5263 R?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚24725955023360 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 5492441088 trainer/global_step‚899 J>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚6346390 NGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚71 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 4552982528 MADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚3324684 H@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚665 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 123731968 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 122905088 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 122905088 MADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚3017412 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚2068772 `ODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 229594510336 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 YFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚24365289308160 F:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚6346390 B6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚6342096 ]JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚24508898246656 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚223 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚209 [JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 216662442377 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 PEDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚760320 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚46132736 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 VCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚24581968789897 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚262 G@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚59 J>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚2829090 ^KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚21378136834048 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚41947648 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 ]JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚24365306347520 XGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 217115829248 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 4552982528 TCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 217073881600 VCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚24726014075904 YFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚24508881141760 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 123731968 % _timestamp‚1.7866588940219204e+09 DDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 I=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚3324684 QEDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚3021704 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 5492441088 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚4294 K>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59052544 F:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚6342096 WDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚21607685211648 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521202 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 4517085184 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚262 I=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚3017412 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 SFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚66776576 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 5368709120 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚321 EDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 _runtime‚ 242.552297353 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 123731968 IBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚59 S@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚24726014075904 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17104896 WFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 217073881600 [HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚21607751988224 bODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚21378157477888 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚59395584 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 5368709120 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚4292Å€C^ž&£˜M TIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚760391 D=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚59 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 4654601728 B9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚5263 MADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚3021704 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654601728 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚321 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17104896 E5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5263 R?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚24725955023360 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 5492441088 trainer/global_step‚899 J>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚6346390 NGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚71 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 4552982528 MADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚3324684 H@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚665 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 123731968 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 122905088 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 122905088 MADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚3017412 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚2068772 `ODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 229594510336 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 YFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚24365289308160 F:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚6346390 B6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚6342096 ]JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚24508898246656 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚223 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚209 [JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 216662442377 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 PEDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚760320 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚46132736 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 VCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚24581968789897 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚262 G@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚59 J>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚2829090 ^KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚21378136834048 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚41947648 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 ]JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚24365306347520 XGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 217115829248 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 4552982528 TCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 217073881600 VCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚24726014075904 YFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚24508881141760 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 123731968 % _timestamp‚1.7866588940219204e+09 DDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 I=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚3324684 QEDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚3021704 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 5492441088 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚4294 K>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59052544 F:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚6342096 WDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚21607685211648 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521202 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 4517085184 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚262 I=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚3017412 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 SFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚66776576 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 5368709120 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚321 EDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 _runtime‚ 242.552297353 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 123731968 IBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚59 S@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚24726014075904 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17104896 WFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 217073881600 [HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚21607751988224 bODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚21378157477888 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚59395584 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 5368709120 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚4292 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚41947648 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚4292 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚2068770 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 843096576 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚321 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 ZGDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚24508898246656 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚20643840 \KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 229548377600 NADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚58387424 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 122306522 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚321 VCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚24508881141760 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚41348064 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 831258624 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 MADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚3324686 G@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚73 R?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚24581910402473 _step‚72H½|ð’0¤+ _wandbruntime‚242  _runtime‚242Þe%m¥ç 0loss/FRt1/assignment_loss‚0.9144217854990045 /loss/FRt1/detection_loss‚0.5583241131256109 %loss/total_loss‚2.944014798449067 epoch‚2 trainer/global_step‚899 _runtime‚ 242.552797918 _step‚73 /loss/FRt2/assignment_loss‚0.910139401791957 /loss/FRt2/detection_loss‚0.5611294980324945 % _timestamp‚1.7866588940229754e+09ÐŽ Iq¦ë 0loss/FRt1/assignment_loss‚0.9144217854990045 /loss/FRt1/detection_loss‚0.5583241131256109 %loss/total_loss‚2.944014798449067 epoch‚2 trainer/global_step‚899 _runtime‚ 242.552797918 _step‚73 /loss/FRt2/assignment_loss‚0.910139401791957 /loss/FRt2/detection_loss‚0.5611294980324945 % _timestamp‚1.7866588940229754e+09I3Ÿ!0§+ _wandbruntime‚254  _runtime‚254L^ðĨ¾ & lr-AdamW/pg2‚0.0014970980942846696 trainer/global_step‚949 % _timestamp‚1.7866589064256165e+09 _runtime‚ 254.955629273 _step‚74 & lr-AdamW/pg1‚0.0014970980942846696õQØPÈ©Â & lr-AdamW/pg1‚0.0014970980942846696 & lr-AdamW/pg2‚0.0014970980942846696 trainer/global_step‚949 % _timestamp‚1.7866589064256165e+09 _runtime‚ 254.955629273 _step‚74J)²~Ë0ª+ _wandbruntime‚254  _runtime‚254Næ&º'«‰O SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 4552982528 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚41944576 IBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚65 UHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚68876800 _step‚75 OCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚3183513 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 K?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚3179227 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17104896 HDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 4552982528 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 OCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚3498781 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 \IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚25798692798464 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 122905088 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚429 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 831258624 BDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 ]LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 228471523328 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚48232960 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 5492441088 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚262 _LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚25798692798464 YHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 228429578752 PDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚2977904 OCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚3179227 KBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚4875 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 4654601728 VEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 228429578752 XEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚26027164321792 ]JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚22749122050048 ]LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 227993680325 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 122306522 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 5368709120 TADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚25876345523733 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4288 C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚4288 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 4517085184 ZIDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 228471523328 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚41944576 PCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚59049472 VKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚801087 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 KDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚59 PIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚63 YHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 227952332309 L@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚2977839 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚2176817 ^MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 241595072000 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚4286 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 4654601728 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚4286 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚321 JBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚665 SGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚3183513 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚20643840 $ _timestamp‚1.786658906426634e+09 XEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚25798675693568 YFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚22749053173248 `MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚22507458101248 bQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 241643304960 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 123731968 @7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚5263 G>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4875 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17104896 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 122905088 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 _LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚25648410230784 TADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚26027105272320 Q>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚26027105272320 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚321 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚321 XEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚25876403911109 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚41348016 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 123731968 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 843096576 dQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚22507478745088 PCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚58387376 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 4628521202 HDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 4552982528 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 OCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚3498781 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 \IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚25798692798464 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 122905088 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚429 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 831258624 BDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 ]LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 228471523328 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚48232960 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 5492441088 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚262 _LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚25798692798464 YHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 228429578752 PDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚2977904 OCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚3179227 KBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚4875 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 4654601728 VEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 228429578752 XEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚26027164321792 ]JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚22749122050048 ]LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 227993680325 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 122306522 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 5368709120 TADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚25876345523733 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4288 C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚4288 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 4517085184 ZIDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 228471523328 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚41944576 PCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚59049472 VKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚801087 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 KDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚59 PIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚63 YHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 227952332309 L@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚2977839 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚2176817 ^MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 241595072000 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚4286 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 4654601728 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚4286 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚321 JBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚665 SGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚3183513 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚20643840 $ _timestamp‚1.786658906426634e+09 XEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚25798675693568 YFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚22749053173248 `MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚22507458101248 bQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 241643304960 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 123731968 @7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚5263 G>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4875 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17104896 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 122905088 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 _LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚25648410230784 TADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚26027105272320 Q>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚26027105272320 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚321 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚321 XEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚25876403911109 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚41348016 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 123731968 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 843096576 dQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚22507478745088 PCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚58387376 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 4628521202 HDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚2980902 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚59395584 YFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚25674578984960 B6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚6684848 VCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚25824998031360 NGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚73 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚209 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚321 trainer/global_step‚949 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 4552982528 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚2179020 R?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚25902761889881 F:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚6689142 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654601728 ]JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚25825015136256 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 4517085184 QEDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚3186806 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚321 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 I=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚3502334 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚4292 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚44035584 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 5368709120 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 123731968 _runtime‚ 255.12322133 TCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 228660647424 [JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 228702595072 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 `ODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 241884466176 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚41348064 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚321 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 122905088 YFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚25824998031360 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 123731968 IBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚59 D=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚59 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 VCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚25902820277305 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚262 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 MADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚3186806 ODeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚6689142 B9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚5263 B5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5263 QEDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚3502336 I@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚4875 G@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚75 MADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚3182514 D8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521202 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 5492441088 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 5492441088 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 S@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚26053717731328 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚41947648 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 [JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 228224252985 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 5368709120 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚321 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 WFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 228182904921 R?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚26053658678784 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 I=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚3182514 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 122905088 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17104896 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17104896 F:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚6684848 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 122306522 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚262 TIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚801955 ^KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚22530505474048 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 EDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59052544 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2ÔH‚&±—M WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚59395584 YFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚25674578984960 B6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚6684848 VCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚25824998031360 NGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚73 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚209 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚321 trainer/global_step‚949 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 4552982528 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚2179020 R?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚25902761889881 F:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚6689142 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654601728 ]JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚25825015136256 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 4517085184 QEDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚3186806 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚321 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 I=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚3502334 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚4292 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚44035584 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 5368709120 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 123731968 _runtime‚ 255.12322133 TCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 228660647424 [JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 228702595072 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 `ODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 241884466176 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚41348064 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚321 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 122905088 YFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚25824998031360 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 123731968 IBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚59 D=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚59 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 VCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚25902820277305 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚262 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 MADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚3186806 ODeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚6689142 B9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚5263 B5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5263 QEDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚3502336 I@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚4875 G@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚75 MADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚3182514 D8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521202 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 5492441088 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 5492441088 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 S@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚26053717731328 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚41947648 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 [JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 228224252985 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 5368709120 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚321 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 WFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 228182904921 R?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚26053658678784 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 I=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚3182514 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 122905088 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17104896 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17104896 F:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚6684848 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 122306522 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚262 TIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚801955 ^KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚22530505474048 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 EDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59052544 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 4654601728 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚41947648 [HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚22772410584064 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚4294 NADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚58387424 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚262 % _timestamp‚1.7866589065929487e+09 ZGDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚25825015136256 EDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚2980902LXü”·0²+ _wandbruntime‚255  _runtime‚255óBqVk³å &loss/total_loss‚3.2222890840258214 epoch‚2 _runtime‚ 255.123770267 _step‚77 trainer/global_step‚949 % _timestamp‚1.7866589065939672e+09 /loss/FRt1/assignment_loss‚1.029895815319342 .loss/FRt2/assignment_loss‚1.04429064687368 .loss/FRt1/detection_loss‚0.586060609516008 /loss/FRt2/detection_loss‚0.5620420123167913g#šo´é /loss/FRt1/assignment_loss‚1.029895815319342 .loss/FRt2/assignment_loss‚1.04429064687368 .loss/FRt1/detection_loss‚0.586060609516008 /loss/FRt2/detection_loss‚0.5620420123167913 &loss/total_loss‚3.2222890840258214 epoch‚2 _runtime‚ 255.123770267 _step‚77 trainer/global_step‚949 % _timestamp‚1.7866589065939672e+09M8QÌ0µ+ _wandbruntime‚266  _runtime‚266O·PÁ¶» & lr-AdamW/pg1‚0.0014965458684345753 & lr-AdamW/pg2‚0.0014965458684345753 trainer/global_step‚999 " _timestamp‚1.7866589184369e+09 _runtime‚ 266.966891635 _step‚78n÷m´Å·¿ & lr-AdamW/pg1‚0.0014965458684345753 & lr-AdamW/pg2‚0.0014965458684345753 trainer/global_step‚999 " _timestamp‚1.7866589184369e+09 _runtime‚ 266.966891635 _step‚78N"t—&0¸+ _wandbruntime‚266  _runtime‚2660%oð¹ŠO TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17104896 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 KDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚59 C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚4288 HDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 Q>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚27354808927744 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 [HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚26957682868224 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚262 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚262 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚41944576 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 5492441088 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚321 IBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚59 UHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚64682496 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 ZIDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 240058289152 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚321 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚4286 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚429 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 PDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚3129319 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 5492441088 F?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚59 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 5368709120 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 122905088 L@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚3129252 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 123731968 XEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚27354867977216 _LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚26957699907584 IBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚67 ]JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚23913768236032 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚20643840 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 122905088 YFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚23913703553536 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 122306522 _step‚79 UBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚27354867977216 \IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚27114809688064 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17104896 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4288 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚2287067 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 5492441088 K?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚3344329 SGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚3676433 ]LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 239555490933 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 123731968 % _timestamp‚1.7866589184377744e+09 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚4286 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 XEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚27197255398517 G>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4875 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 5368709120 trainer/global_step‚999 HDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 4628521202«<¸¢”'ºŽO @7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚5263 F>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 Q>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚27354808927744 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 [HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚26957682868224 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚262 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚262 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚41944576 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 5492441088 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚321 IBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚59 UHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚64682496 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 ZIDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 240058289152 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚321 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚4286 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚429 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 PDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚3129319 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 5492441088 F?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚59 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 5368709120 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 122905088 L@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚3129252 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 123731968 XEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚27354867977216 _LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚26957699907584 IBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚67 ]JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚23913768236032 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚20643840 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 122905088 YFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚23913703553536 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 122306522 _step‚79 UBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚27354867977216 \IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚27114809688064 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17104896 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4288 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚2287067 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 5492441088 K?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚3344329 SGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚3676433 ]LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 239555490933 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 123731968 % _timestamp‚1.7866589184377744e+09 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚4286 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 XEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚27197255398517 G>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4875 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 5368709120 trainer/global_step‚999 HDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 4628521202 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17104896 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 KDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚59 C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚4288 H6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚321 F:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚7031894 NBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚3132366 G@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚70 BDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚223 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚2289272 XGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 240289360896 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 ]JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚26983885701120 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 MADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚3679986 YFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚27141114920960 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 O6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚321 WFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 239744715529 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚2289270 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 $ _timestamp‚1.786658918598236e+09 NADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚59052544 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17104896 QEDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚3679986 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 H@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚665 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 NADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚58387424 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 5368709120 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 TCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 240247413248 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 122905088 \KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 254122686976 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 4517085184 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚262 G@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚59 ]JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚27141132025856 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 trainer/global_step‚999 DDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521202 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 123731968 I@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚4875 NGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚68 [HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚23937063577600 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 5492441088 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚321 D=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚59 >5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5263 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚4292 EDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚7031894 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 4552982528 [JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 240289360896 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 831258624 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚20643840 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 5368709120 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚262 _runtime‚ 267.128505437 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚41348064 B6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚7027600 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 122905088 YFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚26983868661760 [JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 239786063593 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚262 I=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚3679984 R?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚27381362334208 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654601728 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 843096576 ^KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚23682874114048 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 123731968 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 S@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚27381421386752 WFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 240247413248 B9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚5263 WDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚23936996801024 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 5368709120 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚41947648 VCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚27381421386752 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚41947648 MADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚3347616 J>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚3132296 K>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59052544 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 5492441088 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 123731968 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 4654601728 ZGDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚27141132025856 MADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚3679984 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 5492441088 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚4294 I=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚3347616 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚209 F:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚7027600 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 _step‚804Ù`œà½—M K>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59052544 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 5492441088 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 123731968 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 4654601728 ZGDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚27141132025856 MADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚3679984 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 5492441088 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚4294 I=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚3347616 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚209 F:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚7027600 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 _step‚80 QEDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚3351908 `ODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 254168819712 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚321 F:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚7031894 NBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚3132366 G@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚70 BDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚223 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚2289272 XGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 240289360896 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 ]JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚26983885701120 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 MADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚3679986 YFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚27141114920960 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 O6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚321 WFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 239744715529 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚2289270 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 $ _timestamp‚1.786658918598236e+09 NADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚59052544 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17104896 QEDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚3679986 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 H@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚665 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 NADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚58387424 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 5368709120 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 TCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 240247413248 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 122905088 ¤mÄu½\KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 254122686976 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 4517085184 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚262 G@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚59 ]JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚27141132025856 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 trainer/global_step‚999 DDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521202 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 123731968 I@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚4875 NGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚68 [HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚23937063577600 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 5492441088 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚321 D=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚59 >5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5263 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚4292 EDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚7031894 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 4552982528 [JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 240289360896 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 831258624 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚20643840 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 5368709120 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚262 _runtime‚ 267.128505437 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚41348064 B6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚7027600 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 122905088 YFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚26983868661760 [JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 239786063593 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚262 I=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚3679984 R?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚27381362334208 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654601728 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 843096576 ^KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚23682874114048 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 123731968 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 S@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚27381421386752 WFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 240247413248 B9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚5263 WDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚23936996801024 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 5368709120 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚41947648 VCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚27381421386752 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚41947648 MADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚3347616 J>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚3132296Póù*0¾+ _wandbruntime‚267  _runtime‚267KÜh¦n¿è 0loss/FRt1/assignment_loss‚1.0291162008703365 /loss/FRt1/detection_loss‚0.5752640278050474 &loss/total_loss‚3.2510370619820215 epoch‚2 trainer/global_step‚999 _runtime‚ 267.128961809 0loss/FRt2/assignment_loss‚1.0866715477166609 /loss/FRt2/detection_loss‚0.5599852855899768 $ _timestamp‚1.786658918599163e+09 _step‚81¡]årÀì trainer/global_step‚999 _runtime‚ 267.128961809 0loss/FRt2/assignment_loss‚1.0866715477166609 /loss/FRt2/detection_loss‚0.5599852855899768 $ _timestamp‚1.786658918599163e+09 _step‚81 0loss/FRt1/assignment_loss‚1.0291162008703365 /loss/FRt1/detection_loss‚0.5752640278050474 &loss/total_loss‚3.2510370619820215 epoch‚2QôÐœæ¡Á‚(:– é€ùÓÕ«ƒá gpu.0.gpu‚63.0 gpu.0.memory‚12, gpu.0.memoryAllocated‚7.0288848876953125) gpu.0.memoryAllocatedBytes‚ 6037766144 gpu.0.temp‚37.0 gpu.0.powerWatts‚184.761' gpu.0.enforcedPowerLimitWatts‚500.0) gpu.0.powerPercent‚36.952200000000005 gpu.0.smClock‚1410 gpu.0.memoryClock‚1593$ gpu.0.correctedMemoryErrors‚1955# gpu.0.uncorrectedMemoryErrors‚0.‡v‚(:ë é€ùÓŽÑÄã" disk.nvme0n1.out‚ 914.279296875 network.recv‚603915$ disk.nvme0c0n1.out‚ 914.279296875 disk.sda.in‚ 2419.06640625 disk.sda2.in‚0 disk.dm-0.out‚ 914.279296875 disk.nvme0c0n1.in‚ 70.703125 disk.nvme0n1.in‚ 70.703125 disk.dm-1.in‚ 2419.06640625 disk.dm-1.out‚4.796875 disk.drbd2165.in‚0 disk.dm-0.in‚ 70.703125$ memory_percent‚64.10834789276123) proc.memory.availableMB‚ 5880.48828125 disk.sda2.out‚0* disk./.usagePercent‚30.607897775085625% disk./.usageGB‚456.07979583740234 disk.sda.out‚4.796875 disk.sda1.in‚0 disk.sda3.out‚4.796875# proc.memory.rssMB‚ 2285.37890625* proc.memory.percent‚13.948845863342285 network.sent‚ 1.328839e+06 disk.sda1.out‚0 disk.drbd2165.out‚0 disk.sda3.in‚ 2419.06640625 cpu‚25.74190851619265 proc.cpu.threads‚17ÚPwˆ0Ã+ _wandbruntime‚278  _runtime‚278ªGS‚Äľ _runtime‚ 278.809207515 _step‚82 & lr-AdamW/pg1‚0.0014959456933383859 & lr-AdamW/pg2‚0.0014959456933383859 trainer/global_step‚1049 $ _timestamp‚1.786658930279197e+09Ôbd9ÈÅ & lr-AdamW/pg1‚0.0014959456933383859 & lr-AdamW/pg2‚0.0014959456933383859 trainer/global_step‚1049 $ _timestamp‚1.786658930279197e+09 _runtime‚ 278.809207515 _step‚82Rjp›Ÿ0Æ+ _wandbruntime‚278  _runtime‚278™+‘'Ç‹O JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚4286 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 PCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚58387376 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚321 _LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚28430926577664 ]LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 251645054976 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚41944576 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 122905088 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚4286 IBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚66 L@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚3280342 YHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 251075953525 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 4654601728 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 123731968 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚48232960 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚41348016 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 5368709120 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 5492441088 M@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚59049472 XEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚28682571632640 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17104896 OCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚3854081 _step‚83 PCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚59049472 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚209 K?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚3854081 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 4628521202 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 5368709120 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 123731968 dQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚24812216025088 KDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚59 HDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 @7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚5263 UBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚28682571632640 OCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚3509431 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚59395584 IBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚59 ^MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 266153104384 trainer/global_step‚1049 OCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚3854083 YHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 251603110400 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚223 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 5368709120 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚262 ZIDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 251645054976 XEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚28430909472768 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 4552982528 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 HDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚28682512583168 KBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚4875 BDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4875 SGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚3513717 ]JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚25078417362432 UHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚68876800 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚321 C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚4288 K?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚3509431 \IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚28430926577664 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17104896 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 4552982528 PDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚3280408 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 843096576 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 `MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚24812195381248 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 122306522 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 5492441088 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 5492441088 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚262 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 XEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚28518106885925 ]LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 251117301541 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 % _timestamp‚1.7866589302800946e+09 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 122905088 D;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚5263 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚429 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 4654601728 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚2397317 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚2397315 RGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚883027 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚321 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 F?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚59 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4288 VKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚883091 bQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 266201337344 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 [HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚28430909472768 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚20643840 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 VEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 251603110400 TADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚28682512583168 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 831258624 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚321 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 L@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚7367800 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 4517085184 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 _LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚28266989584384 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚41944576 SGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚3854083 PIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚64 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 OCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚3513717 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 YFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚25078348485632 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚262š¥¬•'ÈO HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 XEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚28518106885925 ]LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 251117301541 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 % _timestamp‚1.7866589302800946e+09 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 122905088 D;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚5263 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚429 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 4654601728 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚2397317 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚2397315 RGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚883027 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚321 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 F?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚59 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4288 VKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚883091 bQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 266201337344 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 [HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚28430909472768 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚20643840 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 VEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 251603110400 TADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚28682512583168 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 831258624 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚321 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 L@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚7367800 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 4517085184 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 _LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚28266989584384 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚41944576 SGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚3854083 PIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚64 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 OCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚3513717 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 YFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚25078348485632 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚262 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚4286 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 PCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚58387376 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚321 _LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚28430926577664 ]LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 251645054976 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚41944576 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 122905088 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚4286 IBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚66 L@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚3280342 YHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 251075953525 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 4654601728 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 123731968 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚48232960 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚41348016 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 5368709120 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 5492441088 M@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚59049472 XEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚28682571632640 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17104896 OCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚3854081 _step‚83 PCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚59049472 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚209 K?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚3854081 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 4628521202 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 5368709120 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 123731968 dQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚24812216025088 KDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚59 HDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 @7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚5263 UBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚28682571632640 OCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚3509431 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚59395584 IBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚59 ^MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 266153104384 trainer/global_step‚1049 OCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚3854083 YHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 251603110400 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚223 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 5368709120 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚262 ZIDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 251645054976 XEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚28430909472768 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 4552982528 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 HDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚28682512583168 KBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚4875 BDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4875 SGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚3513717 ]JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚25078417362432 UHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚68876800 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚321 C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚4288 K?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚3509431 \IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚28430926577664 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17104896 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 4552982528 PDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚3280408 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 843096576 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 `MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚24812195381248 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 122306522 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 5492441088 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 5492441088 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚262 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2Sº¯§0É+ _wandbruntime‚278  _runtime‚278åJ˜*xÊ•M ZGDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚28457248915456 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 5368709120 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚4292 D=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚59 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚4294 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17104896 J>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚3283380 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚429 PEDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚883860 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 843096576 ^KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚24835242754048 DDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59052544 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 122905088 \KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 266397774848 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚321 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 5492441088 [JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 251876126720 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 122905088 G@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚70 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚209 B6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚7370352 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521202 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚2399520 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 5368709120 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 123731968 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚41947648 I@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚4875 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚2399522 WDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚25101640528896 R?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚28544464864697 E6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚321 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚262 ODeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚262 trainer/global_step‚1049 J>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚7374646 TIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚883928 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 123731968 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 YFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚28293158338560 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚321 ]JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚28457248915456 bODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚24835263397888 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 _runtime‚ 278.971049699 MADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚3857636 VCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚28709125042176 QEDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚3857636 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚262 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 NGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚68 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 NADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚59052544 MADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚3857634 H@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚665 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654601728 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 4552982528 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 VCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚28544523252121 G@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚59 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 R?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚28709065989632 QEDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚3517010 WFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 251306526137 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 MADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚3512718 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚223 B5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5263 [HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚25101707305472 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚4292 YFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚28457231810560 I=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚3512718 NBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚3283450 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 I=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚3857634Œ“©Ÿ&Ë™M =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 IBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚59 >5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5263 [HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚25101707305472 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚4292 YFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚28457231810560 I=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚3512718 NBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚3283450 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 I=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚3857634 ZGDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚28457248915456 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 5368709120 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚4292 D=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚59 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚4294 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17104896 J>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚3283380 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚429 PEDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚883860 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 843096576 ^KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚24835242754048 DDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59052544 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 122905088 \KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 266397774848 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚321 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 5492441088 [JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 251876126720 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 122905088 G@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚70 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚209 B6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚7370352 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521202 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚2399520 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 5368709120 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 123731968 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚41947648 I@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚4875 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚2399522 WDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚25101640528896 R?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚28544464864697 E6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚321 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚262 ODeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚262 trainer/global_step‚1049 J>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚7374646 TIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚883928 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 123731968 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 YFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚28293158338560 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚321 ]JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚28457248915456 bODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚24835263397888 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 _runtime‚ 278.971049699 MADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚3857636 VCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚28709125042176 QEDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚3857636 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚262 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 NGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚68 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 NADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚59052544 MADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚3857634 H@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚665 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654601728 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 4552982528 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 VCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚28544523252121 G@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚59 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 R?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚28709065989632 QEDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚3517010 WFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 251306526137 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 MADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚3512718 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚223 BDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚30010216238592 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 4552982528 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 122306522 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚223 @7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚5263 PCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚58387376 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚321 HDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 JBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚665 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 843096576 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 831258624 BDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 TADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚29838899985957 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 % _timestamp‚1.7866589422027304e+09 K?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚3674533 PCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚59049472 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚209 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 ]JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚26243059044864 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 K?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚4031731 G>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4875 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17104896 L@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚3432000 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 ]LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 262679112149 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 VKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚924500 RGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚924435 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚44038656 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 5368709120 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚262 KDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚59 _runtime‚ 290.73594049 C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚4288 XEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚30010275288064 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚2507565 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 123731968 ]LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 263231820800 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚41944576 HDeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4288 OCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚3674533 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 SGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚4031733 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚321 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 4628521202 XEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚29838958373333 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 5368709120 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 123731968 M@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚59049472 dQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚25964584665088 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚321 XEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚29747026362368 [HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚29747026362368 KBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚4875 YHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 263189876224 ^MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 278430341120 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚59395584 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚262 trainer/global_step‚1099ñ†Ï 8ÖŽO SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 4552982528 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚321 _step‚87 OCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚4031733 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4288 OCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚3674533 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 SGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚4031733 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚321 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 4628521202 XEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚29838958373333 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 5368709120 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 123731968 M@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚59049472 dQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚25964584665088 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚321 XEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚29747026362368 [HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚29747026362368 KBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚4875 YHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 263189876224 ^MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 278430341120 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚59395584 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚262 trainer/global_step‚1099 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17104896 _LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚29747043467264 `MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚25964564021248 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 5492441088 \IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚29747043467264 VEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 263189876224 ZIDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 263231820800 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚41944576 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 UBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚30010275288064 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 4654601728 F?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚59 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 OCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚3678819 Q>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚30010216238592 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 4552982528 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 122306522 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚223 @7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚5263 PCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚58387376 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚321 HDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 JBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚665 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 843096576 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 831258624 BDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 TADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚29838899985957 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 % _timestamp‚1.7866589422027304e+09 K?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚3674533 PCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚59049472 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚209 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 ]JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚26243059044864 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 K?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚4031731 G>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4875 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17104896 L@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚3432000 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 ]LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 262679112149 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 VKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚924500 RGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚924435 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚44038656 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 5368709120 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚262 KDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚59 _runtime‚ 290.73594049 C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚4288 XEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚30010275288064 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚2507565 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 123731968 ]LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 263231820800 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚41944576 HDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 G@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚59 QEDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚4035286 SFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚68873728 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚321 I=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚3677820 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 4552982528 ^KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚25987611394048 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 123731968 QEDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚3682112 BDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 ODeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 DDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59052544 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚41947648 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚321 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654601728 YFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚29602448015360 NADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚59052544 VCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚29773348700160 MADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚4035284 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚20643840 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚321 XGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 263462892544 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚41947648 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 5492441088 D=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚59 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 4654601728 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚223 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚59395584 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚262 _step‚88 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚4292 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 831258624 MADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚4035286 EDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521202 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚2509770 >5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5263 I=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚4035284 [JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 263462892544 J>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚7717398 H@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚665 NGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚71 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚321 J>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚3435013 R?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚29865316352105 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 5492441088 trainer/global_step‚1099 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚429 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 4517085184 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 122905088 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17104896 WDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚26266283423744 NADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚58387424/hÓŸ&Ù™M QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 5492441088 K>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59052544 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚41947648 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚321 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654601728 YFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚29602448015360 NADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚59052544 VCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚29773348700160 MADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚4035284 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚20643840 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚321 XGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 263462892544 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚41947648 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 5492441088 D=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚59 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 4654601728 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚223 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚59395584 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚262 _step‚88 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚4292 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 831258624 MADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚4035286 EDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521202 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚2509770 >5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5263 I=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚4035284 [JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 263462892544 J>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚7717398 H@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚665 NGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚71 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚321 J>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚3435013 R?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚29865316352105 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 5492441088 trainer/global_step‚1099 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚429 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 4517085184 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 122905088 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17104896 WDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚26266283423744 NADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚58387424 % _timestamp‚1.7866589423645566e+09 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 NBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚3435086 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 G@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚59 QEDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚4035286 SFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚68873728 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚321 I=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚3677820 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 4552982528 ^KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚25987611394048 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 123731968 QEDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚3682112 BDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 ODeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 DDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚31337919894016 KBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚4875 BDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4875 SGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚4209383 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 @7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚5263 PCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚58387376 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 _runtime‚ 303.140004705 OCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚3843921 OCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚3839635 ^MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 290727418368 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 5368709120 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 843096576 F?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚59 YHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 274199574741 JBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚665 L@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚3583323 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚2617815 PIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚65 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 [HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚30885551898624 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚4286 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 4654601728 ]LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 27481Õ²SÏ8586624 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 % _timestamp‚1.7866589546086285e+09 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 4552982528 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 XEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚31159809860741 M@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚59049472 ZIDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 274818586624 VEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 274776642048 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 122905088 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚209 TADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚31159751473365 PCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚59049472 L@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚8053304 ]JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚27407724762112 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 \IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚31063160356864 VKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚965573 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚20643840 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚321 HDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 YFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚27407660079616 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚41348016 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 5492441088 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚321 HDeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4288 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 123731968 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 _step‚91 XEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚31337978943488 IBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚67 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 831258624 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 4552982528 bQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 290771457024 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 122905088 TADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚31337919894016 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚223 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 D;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚5263 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚2617817 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 5368709120 K?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚3839635 C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚4288 XEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚31063143251968 dQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚27116953305088 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚262 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 5368709120 OCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚4209381 IBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚59¼ žA•'äO WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17104896 [HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚31063143251968 YHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 274776642048 UHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚64682496 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚44038656 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 D8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚8049016 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17104896 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 Q>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚31337919894016 KBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚4875 BDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4875 SGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚4209383 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 @7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚5263 PCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚58387376 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 _runtime‚ 303.140004705 OCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚3843921 OCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚3839635 ^MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 290727418368 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 5368709120 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 843096576 F?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚59 YHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 274199574741 JBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚665 L@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚3583323 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚2617815 PIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚65 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 [HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚30885551898624 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚4286 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 4654601728 ]LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 274818586624 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 % _timestamp‚1.7866589546086285e+09 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 4552982528 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 XEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚31159809860741 M@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚59049472 ZIDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 274818586624 VEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 274776642048 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 122905088 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚209 TADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚31159751473365 PCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚59049472 L@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚8053304 ]JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚27407724762112 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 \IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚31063160356864 VKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚965573 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚20643840 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚321 HDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 YFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚27407660079616 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚41348016 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 5492441088 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚321 HDeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4288 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 123731968 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 _step‚91 XEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚31337978943488 IBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚67 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 831258624 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 4552982528 bQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 290771457024 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 122905088 TADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚31337919894016 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚223 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 D;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚5263 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚2617817 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 5368709120 K?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚3839635 C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚4288 XEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚31063143251968 dQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚27116953305088 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚262 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 5368709120 OCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚4209381 IBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚59 _LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚30885568937984 ]LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 274240922757 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 5492441088 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚321 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 4654601728 K?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚4209381 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚429 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 _LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚31063160356864 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚41944576 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 123731968 trainer/global_step‚1149 OCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚4209383 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 4628521202 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0[¶¤ck0å+ _wandbruntime‚303  _runtime‚303~JÕ›&æ•M YFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚31089465589760 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 [JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 274471495417 trainer/global_step‚1149 MADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚3847214 TIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚966407 ]JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚31089482694656 `ODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 291015763968 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 122306522 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 123731968 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 122905088 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚20643840 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 NBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚3586429 ^KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚27139980034048 D=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚59 TCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 275007710720 \KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 290969631232 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 5368709120 % _timestamp‚1.7866589547695687e+09 _runtime‚ 303.299986969 E6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚321 H@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚665 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚2620022 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 831258624 G@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚59 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚41947648 [JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 275049658368 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 4517085184 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 5492441088 XGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 275049658368 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 4552982528 G@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚76 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚2620020 NGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚74 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 843096576 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚262 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 4552982528 B9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚5263 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 IBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚59 S@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚31364532353024 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 WFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 274430147353 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 EDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59052544 J>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚8060150 QEDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚3847214 I@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚4875 [HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚27431016441856 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 5492441088 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚262 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚262 WFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 275007710720 VCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚31186226226937 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 4654601728 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 122905088 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚4294 MADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚4212936 I=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚4212934 VCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚31364532353024 J>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚3586353 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 5368709120 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 5368709120 PEDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚966333 >5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5263 DDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654601728 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521202 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚41947648 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚4292 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 5492441088 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17104896 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚321 F:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚8060150 MADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚3842922 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚321 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 _step‚92 I=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚3842922 VCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚31089465589760 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚429 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚46132736 B6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚8055856 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17104896 F:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚8055856 MADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚4212934 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚209 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚321 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 NADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚58387424 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 R?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚31364473300480 bODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚27140000677888 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 123731968 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚223 R?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚31186167839513 YFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚30911737692160 QEDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚4212936 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0cé}a®ç™M 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 5368709120 % _timestamp‚1.7866589547695687e+09 _runtime‚ 303.299986969 E6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚321 H@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚665 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚2620022 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 831258624 G@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚59 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚41947648 [JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 275049658368 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 4517085184 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 5492441088 XGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 275049658368 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 4552982528 G@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚76 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚2620020 NGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚74 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 843096576 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚262 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 4552982528 B9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚5263 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 IBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚59 S@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚31364532353024 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 WFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 274430147353 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 EDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59052544 J>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚8060150 QEDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚3847214 I@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚4875 [HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚27431016441856 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 5492441088 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚262 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚262 WFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 275007710720 VCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚31186226226937 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 4654601728 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 122905088 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚4294 MADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚4212936 I=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚4212934 VCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚31364532353024 J>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚3586353 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 5368709120 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 5368709120 PEDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚966333 >5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5263 DDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654601728 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521202 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚41947648 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚4292 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 5492441088 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17104896 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚321 F:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚8060150 MADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚3842922 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚321 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 _step‚92 I=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚3842922 VCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚31089465589760 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚429 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚46132736 B6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚8055856 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_p饠ñ ool.current‚17104896 F:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚8055856 MADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚4212934 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚209 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚321 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 NADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚58387424 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 R?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚31364473300480 bODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚27140000677888 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 123731968 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚223 R?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚31186167839513 YFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚30911737692160 QEDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚4212936 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 YFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚31089465589760 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 [JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 274471495417 trainer/global_step‚1149 MADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚3847214 TIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚966407 ]JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚31089482694656 `ODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 291015763968 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 122306522 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 123731968 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 122905088 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚20643840 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 NBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚3586429 ^KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚27139980034048 D=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚59 TCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 275007710720 \KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 290969631232\¬Ó¥0è+ _wandbruntime‚303  _runtime‚303 5hnéè _step‚93 0loss/FRt1/assignment_loss‚1.0965041429311606 &loss/total_loss‚3.2836464282264806 /loss/FRt2/assignment_loss‚1.087540512354775 /loss/FRt1/detection_loss‚0.5417233567557864 /loss/FRt2/detection_loss‚0.5578784161847585 epoch‚2 trainer/global_step‚1149 % _timestamp‚1.7866589547706263e+09 _runtime‚ 303.30044251{Ð*Érêì /loss/FRt2/assignment_loss‚1.087540512354775 /loss/FRt1/detection_loss‚0.5417233567557864 /loss/FRt2/detection_loss‚0.5578784161847585 epoch‚2 trainer/global_step‚1149 % _timestamp‚1.7866589547706263e+09 _runtime‚ 303.30044251 _step‚93 0loss/FRt1/assignment_loss‚1.0965041429311606 &loss/total_loss‚3.2836464282264806]•s–”ë‚(:‰ –ùÓ–®üã gpu.0.gpu‚64.0 gpu.0.memory‚12, gpu.0.memoryAllocated‚7.0288848876953125) gpu.0.memoryAllocatedBytes‚ 6037766144 gpu.0.temp‚35.0 gpu.0.powerWatts‚119.48' gpu.0.enforcedPowerLimitWatts‚500.0 gpu.0.powerPercent‚23.896 gpu.0.smClock‚1410 gpu.0.memoryClock‚1593$ gpu.0.correctedMemoryErrors‚1955# gpu.0.uncorrectedMemoryErrors‚0öh£xŠì‚(:ÿ –ùÓÌŸ´å* disk./.usagePercent‚30.649979013526373 disk.sda.in‚ 2419.06640625 disk.sda.out‚ 5.66015625 disk.sda1.out‚0 disk.sda2.out‚0 disk.nvme0n1.in‚ 70.80859375 disk.dm-1.out‚ 5.66015625 disk.drbd2165.in‚0 disk.drbd2165.out‚0 disk.dm-0.in‚ 70.80859375! disk.nvme0c0n1.in‚ 70.80859375' disk.nvme0c0n1.out‚1650.35595703125$ memory_percent‚64.11261558532715( proc.memory.availableMB‚ 5879.7890625 proc.cpu.threads‚17 network.sent‚ 1.564378e+06 network.recv‚702836 disk.sda1.in‚0 disk.sda2.in‚0% disk.nvme0n1.out‚1650.35595703125 disk.dm-1.in‚ 2419.06640625" disk.dm-0.out‚1650.35595703125* proc.memory.percent‚13.948869705200195% disk./.usageGB‚456.70683670043945 disk.sda3.in‚ 2419.06640625 disk.sda3.out‚ 5.66015625" proc.memory.rssMB‚ 2285.3828125 cpu‚25.923719313721996=WŠž0í+ _wandbruntime‚315  _runtime‚315G!çÂî¼ % lr-AdamW/pg1‚0.001493857873437898 % lr-AdamW/pg2‚0.001493857873437898 trainer/global_step‚1199 $ _timestamp‚1.786658966675192e+09 _runtime‚ 315.205304062 _step‚94jy@(ÆïÀ trainer/global_step‚1199 $ _timestamp‚1.786658966675192e+09 _runtime‚ 315.205304062 _step‚94 % lr-AdamW/pg1‚0.001493857873437898 % lr-AdamW/pg2‚0.001493857873437898^2‰iU0ð+ _wandbruntime‚315  _runtime‚315f¥›h“'ñO O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 4654601728 C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚4288 OCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚4387033 F>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 OCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚4004737 _LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚32194858614784 IBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚59 XEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚32379260141568 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚4286 % _timestamp‚1.7866589666762726e+09 ]LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 286405352448 JBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚665 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚41944576 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 4628521202 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚321 YHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 286363407872 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚209 ^MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 303017798656 SGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚4387033 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚2728067 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 [HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚32194841575424 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 5492441088 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 123731968 UBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚32665682598912 \IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚32379277246464 PDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚3734994 PIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚76 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 5368709120 D8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚8391768 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 122905088 IBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚78 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 843096576 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 122306522 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚321 G>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4875 bQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 303061837312 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚1006927 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚41944576 KBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚4875 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚2728065 ]JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚28572383782400 dQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚28269321945088 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 5368709120 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚262 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 trainer/global_step‚1199 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚4286 YFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚28572319099904 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚59395584 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 PCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚58387376 ]LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 285802733365 F?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚59 _runtime‚ 315.206582286 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 4552982528 PCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚59049472 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17104896 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 123731968 OCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚4387031 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚321 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 5492441088 K?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚4004737 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 VEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 286363407872 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 4654601728 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 4552982528 SGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚4009023 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 4517085184 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚41348016 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚262 @7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚5263 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 5492441088 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 KDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚59 TADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚32480602960773 K?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚4387031 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17104896 [HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚32379260141568 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4288 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚429 `MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚28269301301248 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚44038656 OCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚4009023 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 YHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 285761385349 M@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚59049472 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚321 Q>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚32665623549440 BDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4875 bQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 303061837312 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚1006927 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚41944576 KBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚4875 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚2728065 ]JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚28572383782400 dQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚28269321945088 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 5368709120 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚262 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 trainer/global_step‚1199 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚4286 YFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚28572319099904 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚59395584 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 PCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚58387376 ]LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 285802733365 F?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚59 _runtime‚ 315.206582286 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 4552982528 PCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚59049472 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17104896 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 123731968 OCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚4387031 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚321 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 5492441088 K?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚4004737 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 VEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 286363407872 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 4654601728 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 4552982528 SGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚4009023 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 4517085184 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚41348016 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚262 @7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚5263 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 5492441088 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 KDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚59 TADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚32480602960773 K?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚4387031 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17104896 [HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚32379260141568 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4288 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚429 `MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚28269301301248 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚44038656 OCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚4009023 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 YHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 285761385349 M@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚59049472 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚321 Q>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚32665623549440 BDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 OCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚4004737 _LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚32194858614784 IBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚59 XEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚32379260141568 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚4286 % _timestamp‚1.7866589666762726e+09 ]LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 286405352448 JBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚665 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚41944576 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 4628521202 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚321 YHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 286363407872 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚209 ^MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 303017798656 SGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚4387033 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚2728067 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 [HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚32194841575424 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 5492441088_”âî0ó+ _wandbruntime‚315  _runtime‚315>º…=ô—M `ODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 303308241408 trainer/global_step‚1199 MADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚4390586 ZGDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚32405599584256 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17104896 S@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚32692236008448 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚1007706 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚20643840 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚321 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚321 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 4654601728 NBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚3738055 ^KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚28292348674048 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚50327040 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 5492441088 I=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚4390584 YFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚32405582479360 [JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 286636424192 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 ]JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚32221044408320 D6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚321 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 123731968 F:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚8402902 MADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚4012316 QEDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚4012316 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚429 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚1007783 XGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 286636424192 EDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59052544 ODeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 122306522 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚262 % _timestamp‚1.7866589668431396e+09 J>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚8402902 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚4292 VCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚32507077714345 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚262 IBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚59 SFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚70970880 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 843096576 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521202 VCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚32692236008448 MADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚4008024 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚209 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚262 SDDeviceStatsMonitor.on_train_M ?a` batch_end/reserved_bytes.large_pool.peak‚ 5368709120 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 D=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚59 _step‚96 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚4294 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 122905088 MADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚4390584 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 I@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚4875 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚321 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 5492441088 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 5368709120 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17104896 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚2730270 [HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚28595677559296 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654601728 WFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 286594476544 J>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚3737976 NGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚77 \KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 303257914368 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 E5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5263 F:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚8398608ào 9¡&õ›M B9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚5263 H@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚665 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 831258624 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 BDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59052544 ODeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 122306522 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚262 % _timestamp‚1.7866589668431396e+09 J>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚8402902 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚4292 VCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚32507077714345 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚262 IBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚59 SFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚70970880 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 843096576 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521202 VCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚32692236008448 MADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚4008024 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚209 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚262 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 5368709120 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 D=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚59 _step‚96 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚4294 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 122905088 MADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚4390584 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 I@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚4875 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚321 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 5492441088 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 5368709120 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17104896 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚2730270 [HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚28595677559296 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654601728 WFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 286594476544 J>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚3737976 NGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚77 \KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 303257914368 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 E5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5263 F:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚8398608 `ODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 303308241408 trainer/global_step‚1199 MADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚4390586 ZGDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚32405599584256 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17104896 S@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚32692236008448 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚1007706 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚20643840 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚321 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚321 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 4654601728 NBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚3738055 ^KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚28292348674048 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚50327040 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 5492441088 I=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚4390584 YFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚32405582479360 [JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 286636424192 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 ]JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚32221044408320 D6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚321 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 123731968 F:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚8402902 MADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚4012316 QEDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚4012316 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚429 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚1007783 XGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 286636424192 EŽ•ü‚(:Š ¥ùÓÉœá gpu.0.gpu‚36.0 gpu.0.memory‚5, gpu.0.memoryAllocated‚7.0288848876953125) gpu.0.memoryAllocatedBytes‚ 6037766144 gpu.0.temp‚36.0 gpu.0.powerWatts‚226.001' gpu.0.enforcedPowerLimitWatts‚500.0 gpu.0.powerPercent‚45.2002 gpu.0.smClock‚1410 gpu.0.memoryClock‚1593$ gpu.0.correctedMemoryErrors‚1955# gpu.0.uncorrectedMemoryErrors‚0À ¤Ë‹ý‚(:€ ¥ùÓçºøâ disk.sda1.in‚0 disk.nvme0n1.in‚ 70.80859375" disk.dm-0.out‚1777.20068359375 disk.sda2.out‚0 disk.dm-1.out‚ 5.94140625$ memory_percent‚64.30361270904541# proc.memory.rssMB‚ 2285.47265625 proc.cpu.threads‚17$ disk./.usageGB‚457.2225151062012 disk.sda3.in‚ 2419.06640625' disk.nvme0c0n1.out‚1777.20068359375 disk.sda.in‚ 2419.06640625 disk.dm-1.in‚ 2419.06640625 disk.dm-0.in‚ 70.80859375) proc.memory.availableMB‚ 5848.49609375* proc.memory.percent‚13.949418067932129 network.sent‚ 1.639468e+06 disk.sda3.out‚ 5.94140625 disk.sda.out‚ 5.94140625% disk.nvme0n1.out‚1777.20068359375 disk.drbd2165.in‚0 disk.drbd2165.out‚0 cpu‚26.032765973297504 disk.sda1.out‚0 disk.sda2.in‚0 network.recv‚736056* disk./.usagePercent‚30.684586624020042! disk.nvme0c0n1.in‚ 70.80859375aé†(0þ+ _wandbruntime‚332  _runtime‚332Bh›vÅÿ¿ & lr-AdamW/pg2‚0.0014930663175971594 trainer/global_step‚1249 % _timestamp‚1.7866589841764293e+09 _runtime‚ 332.706440144 _step‚99 & lr-AdamW/pg1‚0.0014930663175971594Ü ¥ ɀà trainer/global_step‚1249 % _timestamp‚1.7866589841764293e+09 _runtime‚ 332.706440144 _step‚99 & lr-AdamW/pg1‚0.0014930663175971594 & lr-AdamW/pg2‚0.0014930663175971594cILƒˆ0+ _wandbruntime‚332  _runtime‚332°ô”'‚ŽO L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 4654602240 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 4552982528 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚2878931 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚209 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚321 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚2878929 \IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚34086331351040 XEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚34086314246144 OCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚4192029 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 831258624 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚262 PCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚59049984 _LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚34086331351040 Q>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚34388332145152 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 4628521206 HDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚223 dQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚29720960892928 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚321 trainer/global_step‚1249 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚4287 G>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4876 ZIDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 302059844096 SGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚4625289 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚20643840 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 _step‚100 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚1056264 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 5492441088 OCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚4196316 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 843096576 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 5368709120 D8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚8817316 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 122905600 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 122905600 ]JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚30040404595712 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚262 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 YHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 302017899008 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 123731968 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚321 UBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚34388391195136 M@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚59049984 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4289 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚4287 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 L@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚3935127 XEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚34186478299502 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 123731968 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 4552982528 KBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚4876 PIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚66 bQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 319443702784 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚59395584 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 4517085184 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 4654602240 `MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚29720940249088 @7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚5264 ]LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 301430516078 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 123731968 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚321 VEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 302017899008 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚262 _runtime‚ 332.710897182 L@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚8821605 HDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 5368709120 D8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚8817316 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 122905600 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 122905600 ]JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚30040404595712 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚262 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 YHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 302017899008 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 123731968 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚321 UBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚34388391195136 M@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚59049984 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4289 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚4287 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 L@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚3935127 XEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚34186478299502 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 123731968 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 4552982528 KBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚4876 PIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚66 bQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 319443702784 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚59395584 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 4517085184 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 4654602240 `MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚29720940249088 @7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚5264 ]LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 301430516078 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 123731968 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚321 VEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 302017899008 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚262 _runtime‚ 332.710897182 L@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚8821605 HDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚34388332145152 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 4628521206 HDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚223 dQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚29720960892928 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚321 trainer/global_step‚1249 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚4287 G>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4876 ZIDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 302059844096 SGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚4625289 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚20643840 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 _step‚100 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚1056264 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 5492441088dùloŸ0„+ _wandbruntime‚332  _runtime‚332ï¦ý&…—M K>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59053056 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚429 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 NBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚3938211 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚59395584 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654602240 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚4295 G@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚71 `ODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 319691515904 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 5368709120 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚209 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 5492441088 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 NADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚58387428 G@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚59 ]JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚34112653688832 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚321 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 122306526 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚41948160 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 WDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚30063628811264 ^KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚29743987621888 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 5368709120 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚50326528 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 VCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚34212894665698 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 123731968 R?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚34414885551616 [HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚30063699781632 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 843096576 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 123731968 VCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚34112636583936 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚2881136 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 VCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚34414944604672 MADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚4195316 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 EDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521206 QEDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚4199609 bODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚29744008265728 WFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 301619740670 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚262 YFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚34112636583936 J>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚8828451 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚223 TCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 302248967680 I=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚4195316 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚262 MADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚4199609 R?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚34212836278270 >5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5264 I=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚4628840 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 4654602240 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚41948160 E8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 5492441088 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚262 XGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 302290915840 MADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚4628842 [JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 302290915840 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 I@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚4876 \KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 319641189376 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚41348068 DDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚3938140 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚2881134 ]JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚33911233576960 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 5492441088 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 123731968 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 YFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚33911216537600 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 4517085184 IBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚59 D=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚59 NGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚69 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚321 [JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 301661088738 F:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚8828451 ODeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521206 QEDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚4199609 bODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚29744008265728 WFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 301619740670 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚262 YFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚34112636583936 J>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚8828451 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚223 TCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 302248967680 I=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚4195316 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚262 MADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚4199609 R?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚34212836278270 >5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5264 I=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚4628840 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 4654602240 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚41948160 E8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 5492441088 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚262 XGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 302290915840 MADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚4628842 [JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 302290915840 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 I@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚4876 \KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 319641189376 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚41348068 DDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚3938140 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚2881134 ]JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚33911233576960 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 5492441088 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 123731968 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 YFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚33911216537600 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 4517085184 IBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚59 D=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚59 NGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚69 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚321 [JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 301661088738 F:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚8828451 ODeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59053056 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚429 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 NBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚3938211 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚59395584 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654602240 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚4295 G@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚71 `ODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 319691515904 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 5368709120 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚209 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 5492441088 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 NADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚58387428 G@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚59 ]JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚34112653688832 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚321 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 122306526 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚41948160 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 WDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚30063628811264 ^KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚29743987621888 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 5368709120 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚50326528 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 VCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚34212894665698 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 123731968 R?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚34414885551616 [HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚30063699781632 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 843096576 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 123731968 VCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚34112636583936 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚2881136 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 VCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚34414944604672 MADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚4195316eVqä$0‡+ _wandbruntime‚332  _runtime‚332²´ü¥pˆê 0loss/FRt1/assignment_loss‚0.9575154426103847 /loss/FRt1/detection_loss‚0.5611518723540186 /loss/FRt2/detection_loss‚0.5574504372873099 epoch‚3 $ _timestamp‚1.786658984341273e+09 _runtime‚ 332.871075248 _step‚102 0loss/FRt2/assignment_loss‚0.9841765635995504 &loss/total_loss‚3.0602943158512637 trainer/global_step‚1249m?Mrt‰î 0loss/FRt1/assignment_loss‚0.9575154426103847 /loss/FRt1/detection_loss‚0.5611518723540186 /loss/FRt2/detection_loss‚0.5574504372873099 epoch‚3 $ _timestamp‚1.786658984341273e+09 _runtime‚ 332.871075248 _step‚102 0loss/FRt2/assignment_loss‚0.9841765635995504 &loss/total_loss‚3.0602943158512637 trainer/global_step‚1249fJw“¶¡Š‚(:– ´ùÓ„Í…ã gpu.0.gpu‚62.0 gpu.0.memory‚11, gpu.0.memoryAllocated‚7.0288848876953125) gpu.0.memoryAllocatedBytes‚ 6037766144 gpu.0.temp‚37.0 gpu.0.powerWatts‚222.735' gpu.0.enforcedPowerLimitWatts‚500.0) gpu.0.powerPercent‚44.547000000000004 gpu.0.smClock‚1410 gpu.0.memoryClock‚1593$ gpu.0.correctedMemoryErrors‚1955# gpu.0.uncorrectedMemoryErrors‚0ØÎ9á{‹‚(:ð ´ùÓñù±æ disk.sda2.out‚0 disk.dm-1.in‚ 2453.7734375* disk./.usagePercent‚30.735514714821534$ disk./.usageGB‚457.9813804626465$ disk.nvme0c0n1.out‚ 3056.21484375 disk.dm-1.out‚ 6.21484375 disk.drbd2165.in‚0 disk.drbd2165.out‚0 disk.dm-0.in‚ 82.08203125 disk.dm-0.out‚ 3056.21484375 disk.sda.in‚ 2453.7734375 disk.sda1.out‚0 disk.nvme0n1.in‚ 82.08203125" disk.nvme0n1.out‚ 3056.21484375* proc.memory.percent‚13.949418067932129 cpu‚26.075482834639796! disk.nvme0c0n1.in‚ 82.08203125 disk.sda.out‚ 6.21484375$ memory_percent‚64.30320739746094% proc.memory.availableMB‚ 5848.5625# proc.memory.rssMB‚ 2285.47265625 proc.cpu.threads‚17 network.sent‚ 1.715014e+06 network.recv‚767963 disk.sda3.in‚ 2453.7734375 disk.sda3.out‚ 6.21484375 disk.sda1.in‚0 disk.sda2.in‚0ï0Œ+ _wandbruntime‚345  _runtime‚3453DòÆÀ & lr-AdamW/pg1‚0.0014922270359953743 & lr-AdamW/pg2‚0.0014922270359953743 trainer/global_step‚1299 % _timestamp‚1.7866589966733787e+09 _runtime‚ 345.205952675 _step‚103ɾB™ÊŽÄ & lr-AdamW/pg1‚0.0014922270359953743 & lr-AdamW/pg2‚0.0014922270359953743 trainer/global_step‚1299 % _timestamp‚1.7866589966733787e+09 _runtime‚ 345.205952675 _step‚103g@‘£0+ _wandbruntime‚345  _runtime‚345ÿªœ!gŽO \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚20643840 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 IBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚59 K?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚4802937 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 XEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚35402431135744 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚223 YFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚31204972727296 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚321 HDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 SGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚4802939 _step‚104 Q>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚35716035800576 VEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 313604664832 UHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚68876288 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 5492441088 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 5368709120 K?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚4357131 IBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚69 XEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚35507329786910 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚41348020 UBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚35716094850560 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 123731968 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 HDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 PCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚59049984 L@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚4086141 ^MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 331663838208 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 4517085184 OCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚4802939 KBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚4876 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 123731968 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚262 TGDeviceStatsMonitor.on_×Éä¸-train_batch_start/active_bytes.large_pool.current‚17104896 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17104896 [HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚35402431135744 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚4287 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚321 ]LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 312992326686 YHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 312950978666 F?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚59 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚41945088 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 _runtime‚ 345.206058104 _LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚35402448240640 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 122905600 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚1096962 XEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚35716094850560 BDeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4289 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚48232448 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 123731968 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 OCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚4361418 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 4628521206 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 122306526 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚262 C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚4289 M@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚59049984 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚41945088 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 5492441088 PIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚67 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 5368709120 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 831258624 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 _LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚35194337460224 G>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4876 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 122905600 @7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚5264 L@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚9164357 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2Ow0M˜'‘’O PIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚67 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 5368709120 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 831258624 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 _LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚35194337460224 G>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4876 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 122905600 @7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚5264 L@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚9164357 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚20643840 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 IBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚59 K?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚4802937 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 XEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚35402431135744 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚223 YFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚31204972727296 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚321 HDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 SGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚4802939 _step‚104 Q>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚35716035800576 VEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 313604664832 UHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚68876288 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 5492441088 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 5368709120 K?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚4357131 IBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚69 XEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚35507329786910 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚41348020 UBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚35716094850560 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 123731968 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 HDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 PCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚59049984 L@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚4086141 ^MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 331663838208 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 4517085184 OCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚4802939 KBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚4876 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 123731968 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚262 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17104896 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17104896 [HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚35402431135744 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚4287 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚321 ]LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 312992326686 YHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 312950978666 F?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚59 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚41945088 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 _runtime‚ 345.206058104 _LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚35402448240640 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 122905600 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚1096962 XEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚35716094850560 BDeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4289 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚48232448 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 123731968 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 OCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚4361418 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 4628521206 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 122306526 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚262 C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚4289 M@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚59049984 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚41945088 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 5492441088hOÖrh0’+ _wandbruntime‚345  _runtime‚345}ÓNž&“˜M VCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚35428753473536 F:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚9166908 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 D=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚59 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 123731968 DDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 5368709120 NADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚59053056 [JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 313877681664 I@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚4876 J>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚4089155 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 831258624 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚321 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚4293 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚41948160 VCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚35533746153106 NADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚58387428 [JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 313222899346 IBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚59 % _timestamp‚1.7866589968354487e+09 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 5368709120 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17104896 EDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚9171203 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚429 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚1097771 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521206 trainer/global_step‚1299 _runtime‚ 345.365661545 MADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚4360418 R?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚35533687765678 YFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚35220506214400 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 4654602240 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚262 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚20643840 YFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚35428753473536 B9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚5264 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚321 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚2991384 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 R?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚35742589207040 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚46132224 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 WFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 313181551278 K>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59053056 WFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 313835733504 QEDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚4364711 SFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚66776064 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 I=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚4360418 NBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚4089215 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚41348068 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚262 TCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 313835733504 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 122905600 WDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚31228267392000 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 5492441088 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚4293 \KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 331911130112 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚321 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 5492441088 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚1097829 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 4517085184 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 B6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚9166908 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 4552982528 XGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 313877681664 ]JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚35428770578432 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 ]JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚35220523253760 ODeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 123731968 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚223 _step‚105 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 VCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚35742648260096 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 123731968 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 QEDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚4806492 ^KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚30896356261888 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚2991386 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 843096576 MADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚4364711 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654602240 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 4552982528 bODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚30896376905728 B5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5264 EDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 123731968 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚223 _step‚105 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 VCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚35742648260096 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 123731968 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 QEDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚4806492 ^KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚30896356261888 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚2991386 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 843096576 MADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚4364711 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654602240 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 4552982528 bODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚30896376905728 B5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5264 EDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 5368709120 NADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚59053056 [JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 313877681664 I@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚4876 J>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚4089155 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 831258624 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚321 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚4293 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚41948160 VCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚35533746153106 NADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚58387428 [JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 313222899346 IBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚59 % _timestamp‚1.7866589968354487e+09 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 5368709120 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17104896 EDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚9171203 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚429 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚1097771 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521206 trainer/global_step‚1299 _runtime‚ 345.365661545 MADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚4360418 R?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚35533687765678 YFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚35220506214400 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 4654602240 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚262 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚20643840 YFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚35428753473536 B9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚5264 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚321 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚2991384 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 R?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚35742589207040 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚46132224 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 WFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 313181551278 K>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59053056 WFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 313835733504 QEDeviceStatsMonitor.on%ÈŽX _train_batch_end/allocation.small_pool.allocated‚4364711 SFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚66776064 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 I=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚4360418 NBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚4089215 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚41348068 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚262 TCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 313835733504 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 122905600 WDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚31228267392000 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 5492441088 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚4293 \KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 331911130112 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚321 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 5492441088 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚1097829 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 4517085184 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 B6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚9166908 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 4552982528 XGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 313877681664 ]JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚35428770578432 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 ]JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚35220523253760 ODeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4289 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚1137917 C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚4289 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 VEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 325191430656 L@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚9507109 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚3099431 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 G>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4876 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 4654602240 _LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚36718565130240 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 4654602240 ]LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 325233375744 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚41945088 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 5368709120 ZIDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 325233375744 PIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚69 BDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 OCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚4980589 HDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚37043739456000 D;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚5264 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 123731968 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 KDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚59 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚1137986 F>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 K?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚4522233 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚223 ]JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚32369694622208 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 % _timestamp‚1.7866590099803424e+09 @7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚5264 KBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚4876 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚321 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚3099429 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 4517085184 K?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚4980587 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17104896)—©²˜'’O SGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚4980589 PDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚4237417 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 843096576 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 831258624 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚262 [HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚36718548025344 L@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚4237346 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 Q>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚37043739456000 D;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚5264 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 123731968 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 KDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚59 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚1137986 F>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 K?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚4522233 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚223 ]JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚32369694622208 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 % _timestamp‚1.7866590099803424e+09 @7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚5264 KBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚4876 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚321 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚3099429 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 4517085184 K?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚4980587 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17104896 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4289 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚1137917 C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚4289 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 VEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 325191430656 L@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚9507109 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚3099431 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 G>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4876 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 4654602240 _LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚36718565130240 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 4654602240 ]LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 325233375744 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚41945088 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 5368709120 ZIDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 325233375744 PIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚69 BDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 OCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚4980589 H6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚321 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 E8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 MADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚4529813 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 4552982528 H@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚665 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚3101634 B6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚9509660 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚321 VCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚37070351915520 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 4552982528 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚429 NGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚84 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 123731968 _runtime‚ 358.690018257 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654602240 [JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 325464447488 G@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚86 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚209 BDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521206 _step‚109 >5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5264 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 122905600 \KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 344184138240 ]JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚36529812930560 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 122306526 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 YFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚36744870363136 VCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚36854597640514 WFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 324743361886 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17104896 [HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚32392986301952 R?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚36854539253086 IBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚59 MADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚4984142 K>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59053056 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 122905600 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 5492441088 R?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚37070292862464 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17104896 MADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚4525520 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚1138802 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚262 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚4293 bODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚32048745545728 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚41348068 G@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚59 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚41948160 SFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚77261824 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚59395584 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 I=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚4984140 ZGDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚36744887468032 MADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚4984140 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚4295 J>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚9513955 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 4517085184 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚321 F:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚9513955 WDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚32392909040128 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚262 S@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚37070351915520 TCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 325422499328 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 831258624 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚321 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚223 E?DeviceStatsMonitor.on_traiÛΑªn_batch_end/active.large_pool.current‚2 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚41948160 WFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 325422499328 F:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚9509660 NADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚58387428 YFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚36529795891200 [JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 324784709954 XGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 325464447488 ]JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚36744887468032 QEDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚4529813 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 123731968 D=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚59 J>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚4240352 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚262 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚4293 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚1138718 QEDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚4984142 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚20643840 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 trainer/global_step‚1349 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0³¦'¢& œM F:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚9513955 WDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚32392909040128 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚262 S@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚37070351915520 TCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 325422499328 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 831258624 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚321 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚223 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚41948160 WFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 325422499328 F:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚9509660 NADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚58387428 YFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚36529795891200 [JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 324784709954 XGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 325464447488 ]JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚36744887468032 QEDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚4529813 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 123731968 D=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚59 J>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚4240352 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚262 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚4293 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚1138718 QEDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚4984142 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚20643840 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 trainer/global_step‚1349 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 B9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚5264 % _timestamp‚1.7866590101595223e+09 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 5368709120 O6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚321 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 E8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 MADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚4529813 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 4552982528 H@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚665 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚3101634 B6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚9509660 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚321 VCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚37070351915520 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 4552982528 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚429 NGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚84 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 123731968 _runtime‚ 358.690018257 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654602240 [JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 325464447488 G@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚86 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚209 BDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521206 _step‚109 >5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5264 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 122905600 \KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 344184138240 ]JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚36529812930560 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 122306526 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 YFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚36744870363136 VCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚36854597640514 WFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 324743361886 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17104896 [HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚32392986301952 R?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚36854539253086 IBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚59 MADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚4984142 K>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59053056 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 122905600 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 5492441088 R?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚37070292862464 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17104896 MADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚4525520 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚1138802 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚262 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚4293 bODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚32048745545728 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚41348068 G@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚59 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚41948160 SFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚77261824 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚59395584 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 I=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚4984140 ZGDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚36744887468032 MADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚4984140 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚4295 J>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚9513955 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 4517085184 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚321mZÖq…0¡+ _wandbruntime‚358  _runtime‚358«”]šq¢ë 0loss/FRt1/assignment_loss‚1.0287811827829625 /loss/FRt1/detection_loss‚0.5498221895138351 /loss/FRt2/detection_loss‚0.5333957995240661 &loss/total_loss‚3.1241910535585076 epoch‚3 % _timestamp‚1.7866590101606185e+09 0loss/FRt2/assignment_loss‚1.0121918817376439 trainer/global_step‚1349 _runtime‚ 358.690466874 _step‚110)>Ÿdu£ï &loss/total_loss‚3.1241910535585076 epoch‚3 % _timestamp‚1.7866590101606185e+09 0loss/FRt2/assignment_loss‚1.0121918817376439 trainer/global_step‚1349 _runtime‚ 358.690466874 _step‚110 0loss/FRt1/assignment_loss‚1.0287811827829625 /loss/FRt1/detection_loss‚0.5498221895138351 /loss/FRt2/detection_loss‚0.5333957995240661nO­ÜŸ¤‚(:” ÃùÓš¦ˆá gpu.0.gpu‚0.0 gpu.0.memory‚0, gpu.0.memoryAllocated‚7.0288848876953125) gpu.0.memoryAllocatedBytes‚ 6037766144 gpu.0.temp‚35.0 gpu.0.powerWatts‚187.374' gpu.0.enforcedPowerLimitWatts‚500.0) gpu.0.powerPercent‚37.474799999999995 gpu.0.smClock‚1410 gpu.0.memoryClock‚1593$ gpu.0.correctedMemoryErrors‚1955# gpu.0.uncorrectedMemoryErrors‚0²4šnŠ¥‚(:ÿ ÃùÓˆúëã disk.dm-1.in‚ 3597.84765625! disk.nvme0c0n1.in‚ 82.08203125 disk.sda.in‚ 3597.84765625% disk.nvme0n1.out‚3064.80029296875 disk.dm-1.out‚ 6.59765625" disk.dm-0.out‚3064.80029296875( proc.memory.availableMB‚ 5848.2421875 network.recv‚789766 disk.sda3.in‚ 3597.84765625 disk.sda.out‚ 6.59765625 disk.sda1.out‚0 disk.dm-0.in‚ 82.08203125# proc.memory.rssMB‚ 2285.48046875 cpu‚26.111387674970942 proc.cpu.threads‚17 disk.sda3.out‚ 6.59765625 disk.sda1.in‚0 disk.drbd2165.in‚0 disk.drbd2165.out‚0$ memory_percent‚64.30516242980957) proc.memory.percent‚13.94946575164795 disk.sda2.in‚0 disk.sda2.out‚0 network.sent‚ 1.779337e+06* disk./.usagePercent‚30.735511898738654% disk./.usageGB‚457.98133850097656' disk.nvme0c0n1.out‚3064.80029296875 disk.nvme0n1.in‚ 82.08203125™aí¿0¦+ _wandbruntime‚370  _runtime‚370‹—ƧÀ & lr-AdamW/pg1‚0.0014904055141924943 & lr-AdamW/pg2‚0.0014904055141924943 trainer/global_step‚1399 % _timestamp‚1.7866590223585358e+09 _runtime‚ 370.888536122 _step‚111ŒmY欀 & lr-AdamW/pg2‚0.0014904055141924943 trainer/global_step‚1399 % _timestamp‚1.7866590223585358e+09 _runtime‚ 370.888536122 _step‚111 & lr-AdamW/pg1‚0.0014904055141924943oIÙ‡0©+ _wandbruntime‚370  _runtime‚370,VÅ~”'ªŽO UHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚66779136 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 831258624 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 `MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚33178046169088 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 4552982528 ]LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 336820141568 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 4517085184 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚321 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 F>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚262 YFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚33534259716608 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚41348020 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 122905600 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚3209681 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 5492441088 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 4654602240 _LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚38034682019840 [HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚37812899774464 _step‚112 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚223 _LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚37812916813824 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚41945088 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 TADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚38148974374346 SGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚4691622 PDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚4388723 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 4628521206 KDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚59 HDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚38371443111424 L@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚9849861 JBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚665 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚3209679 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚1178967 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 5492441088 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17104896 XEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚38034664914944 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚262 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 YHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 336074599882 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚321 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 M@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚59049984 L@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚4388646 PCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚58387380 % _timestamp‚1.7866590223594203e+09 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17104896 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 5368709120 bQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 356259682816 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 123731968 trainer/global_step‚1399 D;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚5264 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚429 ]JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚33534326495744 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚321 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 123731968 F?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚59 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4289 OCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚4687335 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 G>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4876 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚41945088 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚262 YHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 336778196480 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚209 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚4287 PCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚59049984 IBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚77 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 ]LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 336115947902 HDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚38371443111424 L@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚9849861 JBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚665 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚3209679 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚1178967 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 5492441088 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17104896 XEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚38034664914944 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚262 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 YHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 336074599882 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚321 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 M@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚59049984 L@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚4388646 PCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚58387380 % _timestamp‚1.7866590223594203e+09 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17104896 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 5368709120 bQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 356259682816 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 123731968 trainer/global_step‚1399 D;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚5264 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚429 ]JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚33534326495744 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚321 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 123731968 F?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚59 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4289 OCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚4687335 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 G>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4876 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚41945088 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚262 YHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 336778196480 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚209 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚4287 PCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚59049984 IBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚77 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 ]LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 336115947902 HDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚262p7Ð00¬+ _wandbruntime‚371  _runtime‚371ts¾Kž&­˜M I=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚5161790 NBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚4391734 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 123731968 F:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚9856707 SFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚64678912 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 _runtime‚ 371.048896142 MADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚5161790 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 5368709120 O5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5264 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654602240 J>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚9856707 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚209 WDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚33557551923712 IBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚59 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 122905600 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚429 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 831258624 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 5368709120 D=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚59 I=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚4690622 VCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚38060987252736 [JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 337051213312 B9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚5264 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚3211884 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 4552982528 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚4293 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚1179776 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 R?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚38175390740494 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚321 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚262 ZGDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚38061004357632 EDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59053056 [HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚33557616602624 B6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚321 EDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚41948160 H@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚665 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚3211886 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚59395584 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 123731968 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚262 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 B6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚9852412 ]JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚38061004357632 F:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚9852412 J>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚4391660 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚321 G@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚59 R?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚38397996517888 YFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚38060987252736 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521206 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 5492441088 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 VCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚38398055570944 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 4552982528 ^KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚33201093541888 VCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚38175449127922 [JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 336346520562 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚321 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚262 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 5368709120 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 4654602240 TCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 337009265152 NADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚59053056 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚223 \KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 356458381824 ]JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚37839102607360 bODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚33201114185728 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 122306526 S@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚38398055570944 I@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚4876 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚1179848 NGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚72 `ODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 356502416896 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 123731968¨±=t¢&®œM bODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚33201114185728 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 122306526 S@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚38398055570944 I@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚4876 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚1179848 NGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚72 `ODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 356502416896 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 123731968 I=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚5161790 NBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚4391734 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 123731968 F:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚9856707 SFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚64678912 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 _runtime‚ 371.048896142 MADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚5161790 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 5368709120 O5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5264 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654602240 J>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚9856707 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚209 WDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚33557551923712 IBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚59 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 122905600 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚429 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 831258624 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 5368709120 D=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚59 I=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚4690622 VCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚38060987252736 [JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 337051213312 B9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚5264 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚3211884 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 4552982528 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚4293 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚1179776 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 R?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚38175390740494 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚321 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚262 ZGDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚38061004357632 EDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59053056 [HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚33557616602624 B6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚321 EDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚41948160 H@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚665 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚3211886 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚59395584 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 123731968 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚262 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 B6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚9852412 ]JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚38061004357632 F:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚9852412 J>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚4391660 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚321 G@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚59 R?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚38397996517888 YFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚38060987252736 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521206 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 5492441088 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 VCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚38398055570944 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 4552982528 ^KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚33201093541888 VCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚38175449127922 [JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 336346520562 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚321 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚262 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 5368709120 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 4654602240 TCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 337009265152 NADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚59053056 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚223 \KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 356458381824 ]JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚37839102607360q˜[‹0¯+ _wandbruntime‚371  _runtime‚371$Âo°é /loss/FRt1/detection_loss‚0.5459821763810119 -loss/FRt2/detection_loss‚0.53057015947808 trainer/global_step‚1399 _runtime‚ 371.049382179 _step‚114 0loss/FRt1/assignment_loss‚0.9152113279915861 0loss/FRt2/assignment_loss‚0.9194831359433305 &loss/total_loss‚2.9112467997940086 epoch‚3 % _timestamp‚1.7866590225195806e+09±ûƒs±í 0loss/FRt1/assignment_loss‚0.9152113279915861 0loss/FRt2/assignment_loss‚0.9194831359433305 &loss/total_loss‚2.9112467997940086 epoch‚3 % _timestamp‚1.7866590225195806e+09 /loss/FRt1/detection_loss‚0.5459821763810119 -loss/FRt2/detection_loss‚0.53057015947808 trainer/global_step‚1399 _runtime‚ 371.049382179 _step‚114r~{Ú§–²‚(:‹ ÒùÓÆá˜á gpu.0.gpu‚63.0 gpu.0.memory‚12, gpu.0.memoryAllocated‚7.0288848876953125) gpu.0.memoryAllocatedBytes‚ 6037766144 gpu.0.temp‚36.0 gpu.0.powerWatts‚213.914' gpu.0.enforcedPowerLimitWatts‚500.0 gpu.0.powerPercent‚42.7828 gpu.0.smClock‚1410 gpu.0.memoryClock‚1593$ gpu.0.correctedMemoryErrors‚1955# gpu.0.uncorrectedMemoryErrors‚0è`x³‚(:í ÒùÓˆ”Ûã disk.dm-0.in‚83.28125' disk.nvme0c0n1.out‚3070.95263671875 disk.sda1.in‚0# proc.memory.rssMB‚ 2285.48046875) proc.memory.percent‚13.94946575164795 cpu‚26.149209819096537 network.recv‚813781$ disk./.usageGB‚456.8586540222168" disk.dm-0.out‚3070.95263671875 disk.sda3.in‚ 3624.47265625 disk.sda3.out‚6.84375 disk.sda.out‚6.84375 disk.sda2.in‚0 disk.sda2.out‚0 network.sent‚ 1.866373e+06 disk.dm-1.out‚6.84375 disk.drbd2165.in‚0 disk.nvme0c0n1.in‚83.28125 disk.nvme0n1.in‚83.28125% disk.nvme0n1.out‚3070.95263671875$ memory_percent‚64.30656909942627) proc.memory.availableMB‚ 5848.01171875* disk./.usagePercent‚30.660167601374066 disk.sda.in‚ 3624.47265625 disk.sda1.out‚0 proc.cpu.threads‚17 disk.dm-1.in‚ 3624.47265625 disk.drbd2165.out‚0»eb0´+ _wandbruntime‚383  _runtime‚383s8pôĵ¾ % lr-AdamW/pg2‚0.001489423390984341 trainer/global_step‚1449 % _timestamp‚1.7866590353419666e+09 _runtime‚ 383.872045719 _step‚115 % lr-AdamW/pg1‚0.001489423390984341Ûj®Âȶ % lr-AdamW/pg1‚0.001489423390984341 % lr-AdamW/pg2‚0.001489423390984341 trainer/global_step‚1449 % _timestamp‚1.7866590353419666e+09 _runtime‚ 383.872045719 _step‚115sxé60·+ _wandbruntime‚383  _runtime‚383EWŒ !¸’O Q>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚39699146766848 _LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚39350798909440 BDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚10192613 TADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚39469825861754 YHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 347636410490 _runtime‚ 383.876936274 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 843096576 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 5492441088 TADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚39699146766848 L@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚4539847 IBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚59 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚41945088 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 UHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚64681984 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 PCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚58387380 C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚4289 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 K?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚5335887 F>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚41945088 _LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚39122206490624 OCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚4856724 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 122905600 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚321 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚321 \IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚39350798909440 [HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚39350781804544 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 5368709120 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚59395584 JBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚665 ]LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 347677758510 F?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚59 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 4552982528 K?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚4852437 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 122306526 YHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 348364962304 OCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚5335887 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 PCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚59049984 @7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚5264 ^MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 368481026048 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚4287 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚223 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 4654602240 SGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚5335889 KDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚59 trainer/global_step‚1449 % _timestamp‚1.7866590353430037e+09 dQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚34330435452928 `MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚34330414809088 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 XEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚39469884249134 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 G>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4876 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4289 IDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚39699146766848 _LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚39350798909440 BDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚10192613 TADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚39469825861754 YHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 347636410490 _runtime‚ 383.876936274 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 843096576 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 5492441088 TADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚39699146766848 L@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚4539847 IBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚59 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚41945088 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 UHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚64681984 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 PCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚58387380 C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚4289 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 K?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚5335887 F>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚41945088 _LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚39122206490624 OCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚4856724 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 122905600 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚321 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚321 \IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚39350798909440 [HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚39350781804544 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 5368709120 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚59395584 JBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚665 ]LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 347677758510 F?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚59 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 4552982528 K?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚4852437 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 122306526 YHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 348364962304 OCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚5335887 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 PCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚59049984 @7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚5264 ^MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 368481026048 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚4287 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚223 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 4654602240 SGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚5335889 KDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚59 trainer/global_step‚1449 % _timestamp‚1.7866590353430037e+09 dQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚34330435452928 `MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚34330414809088 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 XEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚39469884249134 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 G>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4876 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4289 IDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 123731968 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 4552982528 trainer/global_step‚1449 DDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59053056 VCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚39725759226368 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚10195164 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚4293 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚429 S@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚39725759226368 G@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚71 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚209 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚262 WFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 347866983102 _runtime‚ 384.04993053 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 831258624 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚10199459 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 4552982528 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 YFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚39148375244800 XGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 348637979136 [JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 348637979136 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 123731968 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚321 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚321 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 EDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚10199459 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚44035072 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521206 J>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚4542856 [HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚34722253605888 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚41348068 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 5368709120 ]JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚39377121247232 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 4654602240 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 5492441088 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 5368709120 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 NADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚58387428 G@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚59 MADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚4860017 % _timestamp‚1.7866590355193672e+09 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 D=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚59 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚4293 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 ]JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚39148392284160 [JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 347908331170 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 5492441088 IBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚59 TCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 348596030976 E5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5264 I=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚4855724 O6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚321 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 _step‚117 NGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚69 ^KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚34353462181888 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚59395584 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 5492441088 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚10195164 MADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚5339442 R?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚39725700173312 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚41948160 B9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚5264 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚223 VCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚39496300615330 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚262 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17104896 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚41948160 H@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚665 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 122306526 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 123731968 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654602240 bODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚34353482825728 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚00QsÚ¥&¼ŸM QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 4552982528 trainer/global_step‚1449 DDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59053056 VCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚39725759226368 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚10195164 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚4293 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚429 S@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚39725759226368 G@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚71 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚209 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚262 WFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 347866983102 _runtime‚ 384.04993053 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 831258624 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚10199459 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 4552982528 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 YFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚39148375244800 XGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 348637979136 [JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 348637979136 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 123731968 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚321 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚321 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 EDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚10199459 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚44035072 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521206 J>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚4542856 [HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚34722253605888 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚41348068 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 5368709120 ]JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚39377121247232 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 4654602240 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 5492441088 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 5368709120 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 NADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚58387428 G@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚59 MADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚4860017 % _timestamp‚1.7866590355193672e+09 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 D=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚59 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚4293 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 ]JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚39148392284160 [JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 347908331170 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 5492441088 IBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚59 TCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 348596030976 E5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5264 I=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚4855724 O6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚321 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 _step‚117 NGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚69 ^KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚34353462181888 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚59395584 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 5492441088 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚10195164 MADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚5339442 R?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚39725700173312 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚41948160 B9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚5264 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚223 VCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚39496300615330 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚262 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17104896 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚41948160 H@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚665 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 122306526 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 123731968 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654602240 bODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚34353482825728 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 I=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚5339440 I@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚4876 NBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚4542927 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 MADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚4855724 SFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚64678912 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 123731968uœ'kÍ0½+ _wandbruntime‚384  _runtime‚384%«Äán¾è $ _timestamp‚1.786659035520538e+09 _step‚118 trainer/global_step‚1449 _runtime‚ 384.050408993 0loss/FRt1/assignment_loss‚1.0913617744972208 0loss/FRt2/assignment_loss‚0.9286245597633165 .loss/FRt1/detection_loss‚0.618109610380917 /loss/FRt2/detection_loss‚0.4953485784731476 %loss/total_loss‚3.133444523114602 epoch‚3Ïj»r¿ì 0loss/FRt1/assignment_loss‚1.0913617744972208 0loss/FRt2/assignment_loss‚0.9286245597633165 .loss/FRt1/detection_loss‚0.618109610380917 /loss/FRt2/detection_loss‚0.4953485784731476 %loss/total_loss‚3.133444523114602 epoch‚3 $ _timestamp‚1.786659035520538e+09 _step‚118 trainer/global_step‚1449 _runtime‚ 384.050408993v×2'À‚(:‹ áùÓòß“á gpu.0.gpu‚63.0 gpu.0.memory‚11, gpu.0.memoryAllocated‚7.0288848876953125) gpu.0.memoryAllocatedBytes‚ 6037766144 gpu.0.temp‚36.0 gpu.0.powerWatts‚203.279' gpu.0.enforcedPowerLimitWatts‚500.0 gpu.0.powerPercent‚40.6558 gpu.0.smClocp.{ok‚1410 gpu.0.memoryClock‚1593$ gpu.0.correctedMemoryErrors‚1955# gpu.0.uncorrectedMemoryErrors‚0M#ÉpÁ‚(:å áùÓ˜ÅÃã disk.sda2.in‚0! disk.nvme0n1.out‚ 3078.3359375 disk.dm-1.out‚ 7.08203125 disk.drbd2165.out‚0 disk.sda.in‚ 3624.47265625 disk.sda2.out‚0 disk.drbd2165.in‚0 disk.dm-0.out‚ 3078.3359375 disk.sda3.out‚ 7.08203125 disk.nvme0c0n1.in‚97.875 cpu‚26.180286522023604 proc.cpu.threads‚17 disk.sda1.in‚0 disk.sda3.in‚ 3624.47265625$ memory_percent‚64.30580615997314) proc.memory.availableMB‚ 5848.13671875* disk./.usagePercent‚30.660167857381605 disk.sda1.out‚0 disk.nvme0n1.in‚97.875 disk.dm-1.in‚ 3624.47265625 disk.dm-0.in‚97.875# disk.nvme0c0n1.out‚ 3078.3359375# proc.memory.rssMB‚ 2285.48046875) proc.memory.percent‚13.94946575164795 network.sent‚ 1.934988e+06 network.recv‚839372% disk./.usageGB‚456.85865783691406 disk.sda.out‚ 7.08203125ýtaô0Â+ _wandbruntime‚396  _runtime‚396Ù–ªÂü _runtime‚ 396.206662885 _step‚119 $ lr-AdamW/pg1‚0.00148839377599351 $ lr-AdamW/pg2‚0.00148839377599351 trainer/global_step‚1499 % _timestamp‚1.7866590476766367e+09_[Ê–ÆÄÀ _runtime‚ 396.206662885 _step‚119 $ lr-AdamW/pg1‚0.00148839377599351 $ lr-AdamW/pg2‚0.00148839377599351 trainer/global_step‚1499 % _timestamp‚1.7866590476766367e+09w‡B10Å+ _wandbruntime‚396  _runtime‚396üŸq—'Æ‘O YHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 359951728128 D;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚5264 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚262 KDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚59 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚429 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 123731968 @7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚5264 bQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 380803905536 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚48232448 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚321 KBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚4876 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚41348020 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 IDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 SGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚5021826 L@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚4691299 `MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚35482783449088 PCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚58387380 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚3430181 [HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚40666898694144 ]LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 359993673216 C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚4289 XEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚41026909472256 OCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚5017539 YFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚35863539122176 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 ]LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 359239569118 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 123731968 E8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚10531076 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚1261120 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 4654602240 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚1261186 BDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚41026850422272 XEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚40790735736542 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚10535365 UHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚68876288 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 122306526 M@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚59049984 TADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚41026850422272 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 [HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚40431479128064 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚262 K?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚5017539 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17104896 PIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚66 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 5492441088 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 123731968 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 122905600 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚41945088 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚209 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 dQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚35482804092928 OCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚5513539 PDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚4691367 YHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 359198221098 _step‚120 OCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚5513537 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚20643840 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚321 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚262 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 4552982528 SGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚5513539 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚4287 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚223 ^MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 380755673088 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 4628521206 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 4517085184 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 5492441088 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 G>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4876 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 4654602240 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4289 K?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚5513537 OCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚5021826 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 5492441088 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 5368709120 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 ]JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚35863607998464 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 831258624 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚321 F?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚59 _runtime‚ 396.210875663 \IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚40666915799040 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 122905600 IBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚68 _LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚40431496167424 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 XEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚40666898694144 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 TADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚407906773491625¦W×›'Ç•O @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 4628521206 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 4517085184 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 5492441088 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 G>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4876 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 4654602240 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4289 K?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚5513537 OCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚5021826 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 5492441088 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 5368709120 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 ]JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚35863607998464 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 831258624 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚321 F?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚59 _runtime‚ 396.210875663 \IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚40666915799040 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 122905600 IBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚68 _LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚40431496167424 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 XEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚40666898694144 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 TADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚40790677349162 YHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 359951728128 D;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚5264 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚262 KDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚59 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚429 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 123731968 @7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚5264 bQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 380803905536 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚48232448 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚321 KBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚4876 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚41348020 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 IDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 SGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚5021826 L@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚4691299 `MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚35482783449088 PCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚58387380 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚3430181 [HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚40666898694144 ]LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 359993673216 C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚4289 XEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚41026909472256 OCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚5017539 YFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚35863539122176 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 ]LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 359239569118 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 123731968 E8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚10531076 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚1261120 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 4654602240 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚1261186 BDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚41026850422272 XEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚40790735736542 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚10535365 UHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚68876288 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 122306526 M@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚59049984 TADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚41026850422272 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 [HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚40431479128064 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚262 K?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚5017539 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17104896 PIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚66 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 5492441088 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 123731968 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 122905600 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚41945088 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚209 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 dQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚35482804092928 OCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚5513539 PDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚4691367 YHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 359198221098 _step‚120 OCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚5513537 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚20643840 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚321 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚262 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 4552982528 SGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚5513539 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚4287 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚223 ^MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 380755673088 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0x5¹Û0È+ _wandbruntime‚396  _runtime‚396Y]¢&ÉœM B6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚321 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚41948160 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 123731968 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚262 >5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5264 MADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚5025119 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚4293 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚3432386 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 831258624 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521206 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 5368709120 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17104896 NADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚59053056 NBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚4694401 NGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚67 ^KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚35505830821888 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚59395584 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 5368709120 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚321 G@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚59 trainer/global_step‚1499 DDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 K>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59053056 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 122905600 MADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚5020826 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚262 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚10537916 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 5492441088 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚262 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚429 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚209 D=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚59 E8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 _runtime‚ 396.374382878 QEDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚5025119 J>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚4694332 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 843096576 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 123731968 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 4552982528 I@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚4876 _step‚121 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654602240 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚321 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚321 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 4552982528 VCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚40817152102738 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 `ODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 381049457664 % _timestamp‚1.7866590478440695e+09 [HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚35886900923392 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 YFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚40457664921600 MADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚5517090 G@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚69 bODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚35505851465728 WFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 359428793710 EDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 TCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 360182796800 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚10542211 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚4293 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚1262015 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 R?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚40817093715310 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 5492441088 [JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 360224744960 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚10537916 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚10542211 I=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚5517090 S@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚41053462881792 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17104896 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚1261948 SFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚68873216Ö¸RgèÊ M M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654602240 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚321 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚321 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 4552982528 VCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚40817152102738 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 `ODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 381049457664 % _timestamp‚1.7866590478440695e+09 [HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚35886900923392 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 YFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚40457664921600 MADeviwÎNä¾"ceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚5517090 G@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚69 bODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚35505851465728 WFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 359428793710 EDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 TCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 360182796800 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚10542211 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚4293 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚1262015 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 R?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚40817093715310 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 5492441088 [JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 360224744960 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚10537916 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚10542211 I=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚5517090 S@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚41053462881792 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17104896 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚1261948 SFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚68873216 B6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚321 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚41948160 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 123731968 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚262 >5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5264 MADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚5025119 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚4293 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚3432386 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 831258624 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521206 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 5368709120 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17104896 NADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚59053056 NBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚4694401 NGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚67 ^KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚35505830821888 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚59395584 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 5368709120 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚321 G@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚59 trainer/global_step‚1499 DDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 K>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59053056 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 122905600 MADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚5020826 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚262 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚10537916 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 5492441088 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚262 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚429 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚209 D=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚59 E8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 _runtime‚ 396.374382878 QEDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚5025119 J>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚4694332 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 843096576 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 123731968 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 4552982528 I@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚4876 _step‚121y2(2`0Ë+ _wandbruntime‚396  _runtime‚396ššpÌê 0loss/FRt2/assignment_loss‚0.8686302331683177 /loss/FRt1/detection_loss‚0.5691138705316023 epoch‚3 % _timestamp‚1.7866590478450596e+09 _runtime‚ 396.374928118 _step‚122 /loss/FRt2/detection_loss‚0.4859418928657009 %loss/total_loss‚2.823955698661351 trainer/global_step‚1499 0loss/FRt1/assignment_loss‚0.9002697020957304tP¯ftÍî _runtime‚ 396.374928118 _step‚122 /loss/FRt2/detection_loss‚0.4859418928657009 %loss/total_loss‚2.823955698661351 trainer/global_step‚1499 0loss/FRt1/assignment_loss‚0.9002697020957304 0loss/FRt2/assignment_loss‚0.8686302331683177 /loss/FRt1/detection_loss‚0.5691138705316023 epoch‚3 % _timestamp‚1.7866590478450596e+09z5}Þ§–΂(:‹ ðùÓäêšá gpu.0.gpu‚59.0 gpu.0.memory‚11, gpu.0.memoryAllocated‚7.0288848876953125) gpu.0.memoryAllocatedBytes‚ 6037766144 gpu.0.temp‚36.0 gpu.0.powerWatts‚201.143' gpu.0.enforcedPowerLimitWatts‚500.0 gpu.0.powerPercent‚40.2286 gpu.0.smClock‚1410 gpu.0.memoryClock‚1593$ gpu.0.correctedMemoryErrors‚1955# gpu.0.uncorrectedMemoryErrors‚0¬87…Ï‚(:ú ðùÓõ¬÷ã$ disk.nvme0n1.out‚3082.5869140625' proc.memory.availableMB‚ 5847.609375* disk./.usagePercent‚30.650823582381832% disk./.usageGB‚456.71942138671875 disk.sda.out‚ 7.1796875 disk.sda1.out‚0$ memory_percent‚64.30902481079102) proc.memory.percent‚13.94946575164795 cpu‚26.21144548723125 disk.drbd2165.in‚0 disk.dm-0.in‚ 106.87890625! disk.dm-0.out‚3082.5869140625" disk.nvme0c0n1.in‚ 106.87890625& disk.nvme0c0n1.out‚3082.5869140625 disk.sda1.in‚0 disk.sda2.out‚0 proc.cpu.threads‚17 network.recv‚871952 disk.dm-1.in‚ 3624.47265625 disk.drbd2165.out‚0 disk.nvme0n1.in‚ 106.87890625# proc.memory.rssMB‚ 2285.48046875 network.sent‚ 2.009927e+06 disk.dm-1.out‚ 7.1796875 disk.sda3.in‚ 3624.47265625 disk.sda3.out‚ 7.1796875 disk.sda.in‚ 3624.47265625 disk.sda2.in‚0Ó6¥10Ð+ _wandbruntime‚408  _runtime‚408“àRµÆÑÀ % _timestamp‚1.7866590604358006e+09 _runtime‚ 408.965857311 _step‚123 & lr-AdamW/pg1‚0.0014873167353502484 & lr-AdamW/pg2‚0.0014873167353502484 trainer/global_step‚1549V™ÊÒÄ % _timestamp‚1.7866590604358006e+09 _runtime‚ 408.965857311 _step‚123 & lr-AdamW/pg1‚0.0014873167353502484 & lr-AdamW/pg2‚0.0014873167353502484 trainer/global_step‚1549{|+.Š0Ó+ _wandbruntime‚408  _runtime‚408È•ÂG—'Ô‘O UBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚42354613127680 Q>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚42354554077696 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚1301953 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 4628521206 YHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 370760031706 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 122306526 ZIDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 371580439040 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 BDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4876 JBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚665 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 843096576 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17104896 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 4552982528 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 123731968 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚262 IDeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4289 K?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚5691187 VEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 371538493952 IBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚67 XEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚42354613127680 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 4517085184 F>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 M@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚59049984 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 831258624 _LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚41740785844224 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚321 KDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚59 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚4287 ^MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 393054106112 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 dQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚36635172732928 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 4654602240 TADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚42354554077696 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 OCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚5691187 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚223 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 XEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚41983015583744 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚41945088 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 _LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚41983032688640 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚59395584 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚321 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 5492441088 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 5492441088 bQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 393098144256 TADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚42111528836570 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚321 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚1301888 F?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚59 SGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚5691189 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚3540429 UHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚64681984 [HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚41740768804864 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 _runtime‚ 408.96988519 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 5368709120 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 123731968 trainer/global_step‚1549 OCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚5691189 YHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 371538493952 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚429 PIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚65 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚209 % _timestamp‚1.7866590604367712e+09 _step‚124 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17104896 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚321 IBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚59 C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚4289 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 122905600 PCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚58387380 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 122905600 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 `MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚36635152089088 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 XEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚42111587223950 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚262 @7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚5264 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 5368709120ÐèÏù›'Õ•O WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17104896 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 4552982528 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 123731968 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚262 IDeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4289 K?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚5691187 VEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 371538493952 IBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚67 XEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚42354613127680 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 4517085184 F>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 M@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚59049984 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 831258624 _LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚41740785844224 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚321 KDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚59 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚4287 ^MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 393054106112 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 dQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚36635172732928 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 4654602240 TADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚42354554077696 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 OCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚5691187 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚223 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 XEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚41983015583744 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚41945088 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 _LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚41983032688640 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚59395584 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚321 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 5492441088 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 5492441088 bQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 393098144256 TADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚42111528836570 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚321 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚1301888 F?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚59 SGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚5691189 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚3540429 UHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚64681984 [HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚41740768804864 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 _runtime‚ 408.96988519 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 5368709120 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 123731968 trainer/global_step‚1549 OCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚5691189 YHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 371538493952 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚429 PIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚65 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚209 % _timestamp‚1.7866590604367712e+09 _step‚124 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17104896 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚321 IBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚59 C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚4289 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 122905600 PCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚58387380 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 122905600 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 `MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚36635152089088 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 XEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚42111587223950 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚262 @7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚5264 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 5368709120 UBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚42354613127680 Q>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚42354554077696 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚1301953 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 4628521206 YHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 370760031706 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 122306526 ZIDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 371580439040 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 BDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4876 JBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚665 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 843096576|»'=0Ö+ _wandbruntime‚409  _runtime‚409”ÈŸB_לM J>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚4845327 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚429 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚321 VCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚42381166537216 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 5492441088 DDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 5492441088 H@DeviceStatsMonitor.on_ÊÓQC#train_batch_end/segment.large_pool.current‚262 SFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚66776064 ZGDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚42009355026432 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 4552982528 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚10884963 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚223 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚1302693 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚262 TCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 371769562624 WFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 371769562624 QEDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚5190221 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚59395584 VCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚42138003590146 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 122306526 >5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5264 ODeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚10880668 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 NADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚58387428 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521206 [JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 371031952386 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 4654602240 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 K>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59053056 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚41948160 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚10880668 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚1302757 NGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚64 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 WFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 370990604318 % _timestamp‚1.7866590606047645e+09 I=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚5694740 MADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚5185928 `ODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 393346120704 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 S@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚42381166537216 B9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚5264 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚3542634 ^KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚36658199461888 \KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 393299988480 R?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚42137945202718 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654602240 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 122905600 E8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 MADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚5694740 [HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚37051566226432 YFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚41766954598400 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 G@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚59 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 123731968 G@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚66 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚209 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚321 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 _runtime‚ 409.135113021 ]JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚42009355026432 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚3542636 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 843096576 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚10884963 XGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 371811510784Ž¥mq¦&Ø M UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚10880668 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚1302757 NGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚64 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 WFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 370990604318 % _timestamp‚1.7866590606047645e+09 I=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚5694740 MADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚5185928 `ODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 393346120704 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 S@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚42381166537216 B9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚5264 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚3542634 ^KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚36658199461888 \KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 393299988480 R?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚42137945202718 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654602240 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 122905600 E8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 MADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚5694740 [HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚37051566226432 YFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚41766954598400 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 G@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚59 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 123731968 G@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚66 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚209 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚321 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 _runtime‚ 409.135113021 ]JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚42009355026432 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚3542636 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 843096576 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚10884963 XGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 371811510784 J>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚4845327 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚429 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚321 VCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚42381166537216 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 5492441088 DDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 5492441088 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚262 SFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚66776064 ZGDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚42009355026432 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 4552982528 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚10884963 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚223 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚1302693 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚262 TCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 371769562624 WFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 371769562624 QEDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚5190221 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚59395584 VCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚42138003590146 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 122306526 >5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5264 ODeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚10880668 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 NADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚58387428 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521206 [JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 371031952386 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 4654602240 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 K>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59053056 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚41948160 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0}ÒÚ0Ù+ _wandbruntime‚409  _runtime‚409lð‰pÚê 0loss/FRt1/assignment_loss‚0.8848116041526409 /loss/FRt2/detection_loss‚0.5178650431856713 %loss/total_loss‚2.887448087180006 epoch‚3 _runtime‚ 409.135688318 0loss/FRt2/assignment_loss‚0.9265216261964556 /loss/FRt1/detection_loss‚0.5582498136452382 trainer/global_step‚1549 % _timestamp‚1.7866590606058795e+09 _step‚126«žÔtÛî 0loss/FRt1/assignment_loss‚0.8848116041526409 /loss/FRt2/detection_loss‚0.5178650431856713 %loss/total_loss‚2.887448087180006 epoch‚3 _runtime‚ 409.135688318 0loss/FRt2/assignment_loss‚0.9265216261964556 /loss/FRt1/detection_loss‚0.5582498136452382 trainer/global_step‚1549 % _timestamp‚1.7866590606058795e+09 _step‚126~Öd/k–Ü‚(:‹ ÿùÓðݤá gpu.0.gpu‚64.0 gpu.0.memory‚12, gpu.0.memoryAllocated‚7.0288848876953125) gpu.0.memoryAllocatedBytes‚ 6037766144 gpu.0.temp‚36.0 gpu.0.powerWatts‚217.337' gpu.0.enforcedPowerLimitWatts‚500.0 gpu.0.powerPercent‚43.4674 gpu.0.smClock‚1410 gpu.0.memoryClock‚1593$ gpu.0.correctedMemoryErrors‚1955# gpu.0.uncorrectedMemoryErrors‚0i/Ý‚(:ô ÿùÓüÒçâ network.sent‚ 2.08329e+06 disk.sda3.in‚ 3624.47265625 disk.dm-0.out‚ 3146.6953125# memory_percent‚64.3078088760376# proc.memory.rssMB‚ 2285.48046875 cpu‚26.239642324366443 proc.cpu.threads‚17* disk./.usagePercent‚30.657654631417962 disk.sda.in‚ 3624.47265625 disk.sda1.in‚0 disk.sda1.out‚0 disk.nvme0n1.in‚ 106.87890625! disk.nvme0n1.out‚ 3146.6953125 disk.dm-1.out‚ 7.61328125 disk.drbd2165.in‚0 network.recv‚903077$ disk./.usageGB‚456.8212089538574" disk.nvme0c0n1.in‚ 106.87890625# disk.nvme0c0n1.out‚ 3146.6953125 disk.sda.out‚ 7.61328125 disk.dm-0.in‚ 106.87890625) proc.memory.availableMB‚ 5847.80859375) proc.memory.percent‚13.94946575164795 disk.sda3.out‚ 7.61328125 disk.sda2.in‚0 disk.sda2.out‚0 disk.dm-1.in‚ 3624.47265625 disk.drbd2165.out‚0m?0Þ+ _wandbruntime‚421  _runtime‚421*ž¥IÆßÀ trainer/global_step‚1599 % _timestamp‚1.7866590731067755e+09 _runtime‚ 421.636902878 _step‚127 & lr-AdamW/pg1‚0.0014861923382308638 & lr-AdamW/pg2‚0.0014861923382308638L¢ûÊàÄ & lr-AdamW/pg1‚0.0014861923382308638 & lr-AdamW/pg2‚0.0014861923382308638 trainer/global_step‚1599 % _timestamp‚1.7866590731067755e+09 _runtime‚ 421.636902878 _step‚127þ÷Õd0á+ _wandbruntime‚421  _runtime‚421hź—'â‘O F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 XEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚43432438711358 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 122905600 SGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚5868839 UHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚66779136 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 843096576 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚262 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 122905600 SGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚5352030 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 4517085184 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚321 KDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚59 trainer/global_step‚1599 K?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚5347743 ZIDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 383167204864 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚321 C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚4289 IDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4876 Q>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚43682257733120 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 4654602240 YFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚38192854731776 _LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚43050075521024 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚262 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 4654602240 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 5492441088 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17104896 BDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚41348020 PDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚4993464 IBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚72 L@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚4993392 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚1342713 dQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚37787541372928 KBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚4876 bQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 405380137984 [HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚43050058481664 OCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚5352030 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 5492441088 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 5368709120 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚4287 JBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚665 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚3650681 `MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚37787520729088 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 123731968 IDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 OCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚5868837 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚4287 TADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚43432380323978 IBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚59 @7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚5264 XEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚43682316783104 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17104896 [HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚43299132473344 D;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚5264 PIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚70 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 123731968 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚321 F?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚59 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 4552982528 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 122306526 M@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚59049984 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 5368709120 \IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚43299149578240 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚223 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 5492441088 _step‚128 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚41945088 $ _timestamp‚1.786659073107804e+09 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 YHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 382321842314 VEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 383125259776 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚11220869 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚3650679 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚262 K?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚5868837 TADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚43682257733120 ]LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 383167204864 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4289 ^MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 405334002688=Û, ã–O PDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚4993464 IBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚72 L@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚4993392 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚1342713 dQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚37787541372928 KBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚4876 bQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 405380137984 [HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚43050058481664 OCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚5352030 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 5492441088 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 5368709120 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚4287 JBDeviceSfLb#tatsMonitor.on_train_batch_start/allocation.large_pool.peak‚665 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚3650681 `MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚37787520729088 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 123731968 IDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 OCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚5868837 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚4287 TADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚43432380323978 IBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚59 @7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚5264 XEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚43682316783104 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17104896 [HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚43299132473344 D;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚5264 PIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚70 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 123731968 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚321 F?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚59 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 4552982528 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 122306526 M@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚59049984 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 5368709120 \IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚43299149578240 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚223 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 5492441088 _step‚128 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚41945088 $ _timestamp‚1.786659073107804e+09 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 YHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 382321842314 VEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 383125259776 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚11220869 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚3650679 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚262 K?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚5868837 TADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚43682257733120 ]LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 383167204864 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4289 ^MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 405334002688 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 XEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚43432438711358 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 122905600 SGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚5868839 UHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚66779136 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 843096576 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚262 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 122905600 SGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚5352030 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 4517085184 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚321 KDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚59 trainer/global_step‚1599 K?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚5347743 ZIDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 383167204864 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚321 C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚4289 IDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4876 Q>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚43682257733120 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 4654602240 YFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚38192854731776 _LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚43050075521024 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚262 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 4654602240 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 5492441088 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17104896 BDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚41348020€N×9s0ä+ _wandbruntime‚421  _runtime‚421øû$¢&åœM \KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 405580934144 trainer/global_step‚1599 ]JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚43325471916032 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚3652884 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 843096576 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 831258624 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 5368709120 ODeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654602240 MADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚5351030 J>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚4996432 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚1343609 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚321 R?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚43708811139584 H@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚665 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 123731968 QEDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚5355323 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚3652886 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚1343548 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 TCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 383356328448 MADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚5355323 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚41348068 % _timestamp‚1.7866590732681015e+09 [JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 382593762994 S@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚43708870192640 E8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 R?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚43458796690126 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 DDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59053056 XGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 383398276608 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 NADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚59053056 WFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 382552414926 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 122306526 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚11227715 EDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚11227715 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚4293 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 123731968 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚4293 I=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚5351030 G@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚63 [HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚38216213714944 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚321 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 4552982528 `ODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 405624969216 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 VCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚43458855077554 IBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚59 NADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚58387428 QEDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚5872392 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚59395584 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 5492441088 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17104896 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 122905600 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚321 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 5492441088 [JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 383398276608 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚11223420 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 5368709120 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17104896 bODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚37810588745728 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚321 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚11223420 VCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚43708870192640 I@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚4876 NGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚61 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 4517085184 >5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5264 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 WFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 383356328448 MADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚5872390 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚44035072 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 _step‚129 NBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚4996495 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚20643840 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚4295 ZGDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚43325471916032 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚429 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 ^KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚37810568101888 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 YFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚43325454811136 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚223 BDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521206 ]JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚43076261314560 G@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚59>”ðº§&æ¡M ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 YFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚43325454811136 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚223 BDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521206 ]JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚43076261314560 G@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚59 \KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 405580934144 trainer/global_step‚1599 ]JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚43325471916032 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚3652884 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 843096576 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 831258624 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 5368709120 ODeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654602240 MADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚5351030 J>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚4996432 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚1343609 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚321 R?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚43708811139584 H@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚665 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 123731968 QEDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚5355323 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚3652886 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚1343548 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 TCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 383356328448 MADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚5355323 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚41348068 % _timestamp‚1.7866590732681015e+09 [JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 382593762994 S@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚43708870192640 E8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 R?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚43458796690126 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 DDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59053056 XGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 383398276608 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 NADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚59053056 WFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 382552414926 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 122306526 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚11227715 EDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚11227715 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚4293 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 123731968 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚4293 I=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚5351030 G@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚63 [HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚38216213714944 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚321 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 4552982528 `ODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 405624969216 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 VCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚43458855077554 IBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚59 NADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚58387428 QEDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚5872392 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚59395584 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 5492441088 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17104896 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 122905600 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚321 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 5492441088 [JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 383398276608 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚11223420 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 5368709120 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17104896 bODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚37810588745728 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚321 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚11223420 VCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚43708870192640 I@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚4876 NGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚61 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 4517085184 >5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5264 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 WFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 383356328448 MADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚5872390 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚44035072 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 _step‚129 NBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚4996495 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚20643840 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚4295 ZGDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚43325471916032 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚429 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 ^KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚37810568101888áʲÈ0ç+ _wandbruntime‚421  _runtime‚421Íë8;pèê 0loss/FRt2/assignment_loss‚0.8852707347433169 /loss/FRt2/detection_loss‚0.5220337477092922 trainer/global_step‚1599 % _timestamp‚1.7866590732690609e+09 _step‚130 0loss/FRt1/assignment_loss‚0.9413114382278102 /loss/FRt1/detection_loss‚0.5430592022236849 %loss/total_loss‚2.891675122904104 epoch‚3 _runtime‚ 421.798927948’quéï _runtime‚ 421.798927948 0loss/FRt2/assignment_loss‚0.8852707347433169 /loss/FRt2/detection_loss‚0.5220337477092922 trainer/global_step‚1599 % _timestamp‚1.7866590732690609e+09 _step‚130 0loss/FRt1/assignment_loss‚0.9413114382278102 /loss/FRt1/detection_loss‚0.5430592022236849 %loss/total_loss‚2.891675122904104 epoch‚3‚£.ö0ê+ _wandbruntime‚426  _runtime‚4269 ’µÚëÔ 3Purity/1FRt/event_proportion‚0.6297432780265808 -Purity/1FRt/FRt_purity‚0.3829471468925476 /particle/accuracy_2_of_2‚0.4655906558036804 'particle/f_score‚0.5194167494773865 2Purity/*FRt/event_proportion‚0.669805645942688 % _timestamp‚1.7866590782225277e+09 _runtime‚ 426.75256564 /Purity/*FRt/event_purity‚0.3784661889076233 3Purity/0FRt/event_proportion‚0.3301943838596344 -Purity/2FRt/FRt_purity‚0.4596826136112213 -Purity/*FRt/FRt_purity‚0.3916615843772888 0Purity/2FRt/event_purity‚0.30694064497947693 _step‚131 Purity/0FRt/event_purity‚1 +jet/accuracy_2_of_2‚0.15671029686927795 +particle/sensitivity‚0.5395533442497253 +particle/specificity‚0.7041115164756775 +validation_accuracy‚0.15671029686927795 epoch‚3 6validation_average_jet_accuracy‚0.2599281072616577 2Purity/2FRt/event_proportion‚0.040062315762043 +jet/accuracy_1_of_2‚0.49382132291793823 particle/accuracy_1_of_1‚1 (particle/accuracy‚0.6456822156906128 /Purity/1FRt/event_purity‚0.3829471468925476 *jet/accuracy_1_of_1‚0.2557763457298279 /particle/accuracy_1_of_2‚0.7973088026046753 trainer/global_step‚1599 )loss/val_total_loss‚2.974314909829908R 6ßìÙ /Purity/1FRt/event_purity‚0.3829471468925476 *jet/accuracy_1_of_1‚0.2557763457298279 /particle/accuracy_1_of_2‚0.7973088026046753 trainer/global_step‚1599 )loss/val_total_loss‚2.974314909829908 3Purity/1FRt/event_proportion‚0.6297432780265808 -Purity/1FRt/FRt_purity‚0.3829471468925476 /particle/accuracy_2_of_2‚0.4655906558036804 'particle/f_score‚0.5194167494773865 2Purity/*FRt/event_proportion‚0.669805645942688 % _timestamp‚1.7866590782225277e+09 _runtime‚ 426.75256564 /Purity/*FRt/event_purity‚0.3784661889076233 3Purity/0FRt/event_proportion‚0.3301943838596344 -Purity/2FRt/FRt_purity‚0.4596826136112213 -Purity/*FRt/FRt_purity‚0.3916615843772888 0Purity/2FRt/event_purity‚0.30694064497947693 _step‚131 Purity/0FRt/event_purity‚1 +jet/accuracy_2_of_2‚0.15671029686927795 +particle/sensitivity‚0.5395533442497253 +particle/specificity‚0.7041115164756775 +validation_accuracy‚0.15671029686927795 epoch‚3 6validation_average_jet_accuracy‚0.2599281072616577 2Purity/2FRt/event_proportion‚0.040062315762043 +jet/accuracy_1_of_2‚0.49382132291793823 particle/accuracy_1_of_1‚1 (particle/accuracy‚0.6456822156906128ƒK)³ –í‚(:‹ Ž‚ùÓ³÷à gpu.0.gpu‚64.0 gpu.0.memory‚12, gpu.0.memoryAllocated‚7.0288848876953125) gpu.0.memoryAllocatedBytes‚ 6037766144 gpu.0.temp‚34.0 gpu.0.powerWatts‚119.733' gpu.0.enforcedPowerLimitWatts‚500.0 gpu.0.powerPercent‚23.9466 gpu.0.smClock‚1410 gpu.0.memoryClock‚1593$ gpu.0.correctedMemoryErrors‚1955# gpu.0.uncorrectedMemoryErrors‚0 ž›{î‚(:÷ Ž}r€Ks‚ùÓʪÑâ) proc.memory.availableMB‚ 5756.30078125 disk.nvme0n1.in‚ 106.87890625 disk.dm-1.in‚ 3624.47265625 disk.dm-0.out‚ 3184.0703125 disk.sda2.out‚0# proc.memory.rssMB‚ 2376.41015625* proc.memory.percent‚14.504456520080566 disk.dm-1.out‚ 8.05078125 disk.sda3.in‚ 3624.47265625 disk.sda3.out‚ 8.05078125" disk.nvme0c0n1.in‚ 106.87890625 disk.sda1.out‚0 disk.sda2.in‚0$ memory_percent‚64.86632823944092 proc.cpu.threads‚17* disk./.usagePercent‚29.648224092119456! disk.nvme0n1.out‚ 3184.0703125 disk.dm-0.in‚ 106.87890625# disk.nvme0c0n1.out‚ 3184.0703125 disk.sda1.in‚0 cpu‚26.321017633556586 network.sent‚ 2.155033e+06 network.recv‚932414$ disk./.usageGB‚441.7799644470215 disk.drbd2165.in‚0 disk.drbd2165.out‚0 disk.sda.in‚ 3624.47265625 disk.sda.out‚ 8.05078125ûžN0ï+ _wandbruntime‚439  _runtime‚439~Ô6¯Åð¿ _step‚132 & lr-AdamW/pg1‚0.0014850206568532821 & lr-AdamW/pg2‚0.0014850206568532821 trainer/global_step‚1649 $ _timestamp‚1.786659090900031e+09 _runtime‚ 439.430080251Z!´^ÊñÄ trainer/global_step‚1649 $ _timestamp‚1.786659090900031e+09 _runtime‚ 439.430080251 _step‚132 & lr-AdamW/pg1‚0.0014850206568532821 & lr-AdamW/pg2‚0.0014850206568532821„ôÜ}…0ò+ _wandbruntime‚439  _runtime‚439{c²Ù—'ó‘O L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 4654602240 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 4552982528 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚321 YHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 398779751936 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚1392548 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 5368709120 K?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚6107093 TADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚45404966329856 _LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚45006203682816 JBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚665 K?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚5535037 XEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚45138255662712 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚41945088 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚44038144 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 PCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚59049984 OCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚6107095 XEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚45006186577920 XEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚45405025379840 PIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚66 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚59395584 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 trainer/global_step‚1649 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚321 UBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚45405025379840 OCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚5535037 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚3801543 YFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚39660881043456 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 4654602240 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 122905600 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚321 IBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚59 _runtime‚ 439.434896585 @7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚5264 [HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚45006186577920 F?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚59 OCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚5539324 VEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 398779751936 IBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚68 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚209 bQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 421765404672 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 4628521206 [HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚44740247650304 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 5368709120 $ _timestamp‚1.786659090900992e+09 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚41945088 IDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚11646419 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚262 KDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚59 PDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚5194093 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚429 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚3801545 _LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚44740264689664 D;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚5264 KBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚4876 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚20643840 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 PCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚58387380 IDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4876 M@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚59049984 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 122905600 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚223 ^MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 421721366528 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 5492441088 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 5368709120 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 \IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚45006203682816 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4289 dQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚39239180320768 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚41348020 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 122306526 C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚4289 Q>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚45404966329856 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 4552982528 BDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚11646419 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚262 KDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚59 PDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚5194093 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚429 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚3801545 _LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚44740264689664 D;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚5264 KBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚4876 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚20643840 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 PCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚58387380 IDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4876 M@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚59049984 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 122905600 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚223 ^MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 421721366528 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 5492441088 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 5368709120 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 \IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚45006203682816 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4289 dQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚39239180320768 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚41348020 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 122306526 C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚4289 Q>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚45404966329856 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 4552982528 B6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚321 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 % _timestamp‚1.7866590910667822e+09 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚4293 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 122905600 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚209 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 4517085184 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚321 _runtime‚ 439.597125666 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚11653265 R?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚45431519736320 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 YFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚44766433443840 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚41948160 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 D6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚321 _step‚134 I=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚5538324 I@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚4876 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 I=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚6110646 O8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 5368709120 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚262 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 YFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚45032508915712 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚1393361 TCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 399010820608 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚11653265 MADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚6110646 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚429 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521206 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚262 VCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚45431578789376 B9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚5264 WDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚39684166959104 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 5368709120 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 5368709120 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚4295 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚11648970 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 bODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚39262227693568 EDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654602240 `ODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 422012333056 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 G@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚59 ]JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚45032526020608 ]JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚44766450483200 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17104896 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚4293 NGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚72 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 VCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚45164672028908 IBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚59 VCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚45032508915712 SFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚73067520 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚11648970 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 123731968 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 4552982528 XGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 399052768768 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17104896 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚20643840 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚3803748 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 4654602240 [JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 399052768768 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 843096576 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚3803750 BDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚5197037 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚1393289 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 >5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5264 K>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59053056 WFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 399010820608 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚223 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 831258624 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚52423680 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 MADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚6110648 E6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚321 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 % _timestamp‚1.7866590910667822e+09 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚4293 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 122905600 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚209 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 4517085184 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚321 _runtime‚ 439.597125666 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚11653265 R?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚45431519736320 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 YFDeviceStatsMonitor.on_train_batch_euåä2T"nd/requested_bytes.large_pool.freed‚44766433443840 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚41948160 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 D6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚321 _step‚134 I=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚5538324 I@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚4876 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 I=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚6110646 O8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 5368709120 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚262 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 YFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚45032508915712 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚1393361 TCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 399010820608 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚11653265 MADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚6110646 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚429 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521206 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚262 VCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚45431578789376 B9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚5264 WDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚39684166959104 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 5368709120 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 5368709120 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚4295 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚11648970 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 bODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚39262227693568 EDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654602240 `ODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 422012333056 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 G@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚59 ]JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚45032526020608 ]JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚44766450483200 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17104896 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚4293 NGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚72 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 VCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚45164672028908 IBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚59 VCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚45032508915712 SFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚73067520 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚11648970 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 123731968 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 4552982528 XGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 399052768768 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17104896 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚20643840 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚3803748 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 4654602240 [JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 399052768768 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 843096576 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚3803750 BDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚5197037 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚1393289 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 >5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5264 K>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59053056 WFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 399010820608 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚223 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 831258624 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚52423680 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 MADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚6110648 EDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 \IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚46322320572416 ZIDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 410408462848 dQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚40391548960768 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚41940992 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚262 KDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚59 IBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚59 IDeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4289 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚3911793 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 XEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚46459107150120 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 5368709120 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚262 PDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚5344932 UBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚46732729035264 JBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚665 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 PCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚58387380 YHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 409511435636 OCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚5704426 K?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚5700139 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 831258624 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 SGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚5704426 `MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚40391528316928 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚321 BDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 UHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚62584832 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚262 L@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚5344865 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚20643840 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 5492441088 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚41945088 YFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚40825530179584 ^MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 434001862656 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 4654602240 @7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚5264 Q>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚46732669985280 OCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚6284743 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 4628521206 _runtime‚ 452.198889571 _step‚137 M@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚59049984 TADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚46732669985280 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 122905600 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 4517085184 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 YHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 410366517760 trainer/global_step‚1699 XEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚46322303467520 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚41945088 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚223 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 123731968 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 ]LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 410408462848 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 E8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚11984882 OCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚6284745 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚429 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 123731968 VEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 410366517760 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚209 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚321 D;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚5264 OCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚5700139 TADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚46459048762740 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 _LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚46322320572416 KBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚4876 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 % _timestamp‚1.7866591036656034e+09 C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚4289 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 XEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚46732729035264 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 4552982528 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 843096576 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 122306526 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚321 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚321 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚4287 G>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4876 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚1433137 [HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚46049537327104 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚1433072 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 5492441088 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 [HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚46322303467520 bQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 434043803648 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚41348020 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 123731968 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚3911795º^uÝ'‚—O SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 5492441088 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚41945088 YFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚40825530179584 ^MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 434001862656 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 4654602240 @7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚5264 Q>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚46732669985280 OCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚6284743 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 4628521206 _runtime‚ 452.198889571 _step‚137 M@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚59049984 TADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚46732669985280 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 122905600 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 4517085184 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 YHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 410366517760 trainer/global_step‚1699 XEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚46322303467520 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚41945088 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚223 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 123731968 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 ]LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 410408462848 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 E8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚11984882 OCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚6284745 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚429 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 123731968 VEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 410366517760 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚209 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚321 D;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚5264 OCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚5700139 TADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚46459048762740 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 _LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚46322320572416 KBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚4876 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 % _timestamp‚1.7866591036656034e+09 C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚4289 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 XEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚46732729035264 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 4552982528 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 843096576 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 122306526 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚321 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚321 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚4287 G>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4876 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚1433137 [HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚46049537327104 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚1433072 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 5492441088 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 [HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚46322303467520 bQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 434043803648 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚41348020 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 123731968 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚3911795 K?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚6284743 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 4654602240 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17104896 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚11989171 PCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚59049984 SGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚6284745 ]JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚40825592764416 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 \IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚46322320572416 ZIDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 410408462848 dQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚40391548960768 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚41940992 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚262 KDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚59 IBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚59 IDeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4289 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚3911793 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 XEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚46459107150120 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 5368709120 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚262 PDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚5344932 UBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚46732729035264 JBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚665 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 PCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚58387380 YHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 409511435636 OCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚5704426 K?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚5700139 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 831258624 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 SGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚5704426 `MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚40391528316928 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚321 BDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 UHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚62584832 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚262 L@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚5344865 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚20643840‰¸\úd0ƒ+ _wandbruntime‚452  _runtime‚452Ê8ç6Ê„œM DDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 122905600 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚59395584 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚262 >5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5264 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 4552982528 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚3914000 [JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 409783356316 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚4295 EDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17104896 YFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚46348625805312 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 R?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚46485465128888 ]JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚46075740160000 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚41348068 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 123731968 [JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 410639534592 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 WFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 409742008248 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 5368709120 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 5368709120 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 123731968 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚41948160 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 123731968 R?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚46759223391744 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚1433933 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 5492441088 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚4293 BDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 SFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚73067520 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 YFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚46075723120640 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚11996017 B9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚5264 MADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚5703426 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚223 ^KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚40414575689728 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 4654602240 J>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚5347862 VCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚46348625805312 MADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚6288296 I=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚6288296 TCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 410597586432 WFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 410597586432 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚11991722 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 5368709120 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚321 NGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚69 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚321 IBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚59 VCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚46759282444800 NBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚5347933 WDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚40848817668096 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 831258624 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚262 bODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚40414596333568 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 K>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59053056 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 D=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚59 trainer/global_step‚1699 E6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚321 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚4293 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521206 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 4517085184 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 5492441088 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚262 G@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚59 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 122905600 QEDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚6288298 [HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚40848890735616 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 5492441088 _runtime‚ 452.363820958 _step‚138 ODeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚11996017 G@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚71 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚321 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654602240 I@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚4876¨Š6W§&…¡M bODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚40414596333568 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 K>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59053056 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 D=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚59 trainer/global_step‚1699 E6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚321 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚4293 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521206 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 4517085184 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 5492441088 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚262 G@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚59 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 122905600 QEDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚6288298 [HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚40848890735616 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 5492441088 _runtime‚ 452.363820958 _step‚138 ODeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚11996017 G@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚71 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚321 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654602240 I@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚4876 DDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 122905600 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚59395584 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚262 >5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5264 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 4552982528 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚3914000 [JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 409783356316 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚4295 EDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17104896 YFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚46348625805312 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 R?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚46485465128888 ]JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚46075740160000 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚41348068 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 123731968 [JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 410639534592 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 WFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 409742008248 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 5368709120 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 5368709120 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 123731968 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚41948160 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 123731968 R?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚46759223391744 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚1433933 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 5492441088 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚4293 BDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 SFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚73067520 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 YFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚46075723120640 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚11996017 B9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚5264 MADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚5703426 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚223 ^KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚40414575689728 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 4654602240 J>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚5347862 VCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚46348625805312 MADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚6288296 I=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚6288296 TCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 410597586432 WFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 410597586432 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚11991722 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 5368709120 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚321 NGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚69 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚321 IBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚59 VCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚46759282444800 NBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚5347933 WDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚40848817668096 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 831258624 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚262Š|s0†+ _wandbruntime‚452  _runtime‚452¹¼ðq‡ë 0loss/FRt2/assignment_loss‚0.8801541939177904 /loss/FRt2/detection_loss‚0.5267366396191548 &loss/total_loss‚2.7896113473361766 trainer/global_step‚1699 % _timestamp‚1.7866591038348384e+09 _runtime‚ 452.364645677 0loss/FRt1/assignment_loss‚0.8493078817340869 /loss/FRt1/detection_loss‚0.5334126320651443 epoch‚4 _step‚139V’ vˆð % _timestamp‚1.7866591038348384e+09 _runtime‚ 452.364645677 0loss/FRt1/assignment_loss‚0.8493078817340869 /loss/FRt1/detection_loss‚0.5334126320651443 epoch‚4 _step‚139 0loss/FRt2/assignment_loss‚0.8801541939177904 /loss/FRt2/detection_loss‚0.5267366396191548 &loss/total_loss‚2.7896113473361766 trainer/global_step‚1699‹&èY{¡‰‚(:– ¬‚ùÓ¬Á£á gpu.0.gpu‚58.0 gpu.0.memory‚10, gpu.0.memoryAllocated‚7.0288848876953125) gpu.0.memoryAllocatedBytes‚ 6037766144 gpu.0.temp‚36.0 gpu.0.powerWatts‚204.395' gpu.0.enforcedPowerLimitWatts‚500.0) gpu.0.powerPercent‚40.879000000000005 gpu.0.smClock‚1410 gpu.0.memoryClock‚1593$ gpu.0.correctedMemoryErrors‚1955# gpu.0.uncorrectedMemoryErrors‚0ØMÁ…Š‚(:ú ¬‚ùÓ« Ûã* proc.memory.percent‚14.504456520080566 network.recv‚995854$ disk./.usageGB‚441.8390769958496 disk.sda3.out‚ 9.10546875 disk.sda.in‚ 3647.33984375 disk.sda1.in‚0 disk.drbd2165.in‚0 cpu‚26.36395967176378) disk./.usagePercent‚29.65219118487005 disk.sda.out‚ 9.10546875 disk.sda1.out‚0 disk.sda2.out‚0 disk.nvme0n1.in‚ 129.8359375$ disk.nvme0n1.out‚3357.4619140625 disk.dm-1.in‚ 3647.33984375 disk.dm-0.in‚ 129.8359375 network.sent‚ 2.304504e+06 disk.sda3.in‚ 3647.33984375! disk.nvme0c0n1.in‚ 129.8359375 disk.drbd2165.out‚0# memory_percent‚64.8674726486206) proc.memory.availableMB‚ 5756.11328125 proc.cpu.threads‚17& disk.nvme0c0n1.out‚3357.4619140625 disk.sda2.in‚0 disk.dm-1.out‚ 9.10546875! disk.dm-0.out‚3357.4619140625# proc.memory.rssMB‚ 2376.41015625ó É0‹+ _wandbruntime‚465  _runtime‚465ƒ 2ÆŒÀ % _timestamp‚1.7866591166197994e+09 _runtime‚ 465.149838862 _step‚140 & lr-AdamW/pg1‚0.0014825357453752959 & lr-AdamW/pg2‚0.0014825357453752959 trainer/global_step‚1749[}/hËÅ % _timestamp‚1.7866591166197994e+09 _runtime‚ 465.149838862 _step‚140 & lr-AdamW/pg1‚0.0014825357453752959 & lr-AdamW/pg2‚0.0014825357453752959 trainer/global_step‚1749ŒC+âÞ0Ž+ _wandbruntime‚465  _runtime‚465ÿ€;%˜'’O ]JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚41990254957056 YFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚41990192372224 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 [HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚47358827003904 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17104896 F?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚59 % _timestamp‚1.7866591166207519e+09 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 4654602240 SGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚5869528 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚1474053 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚41945088 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚321 M@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚59049984 TADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚48060373640704 IDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 4552982528 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚223 YHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 421073246244 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚262 @7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚5264 ]LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 421995228672 YHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 421953283584 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 843096576 ZIDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 421995228672 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 122905600 _LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚47638437462016 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 122905600 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 F>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚4022043 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 5368709120 OCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚6462393 UHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚62584832 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 PCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚58387380 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 5492441088 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 5492441088 trainer/global_step‚1749 IDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4876 OCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚5865241 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 TADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚47779900250148 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 _step‚141 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚4287 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚41945088 D;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚5264 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚4022045 SGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚6462395 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚429 dQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚41543917600768 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚321 OCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚6462395 L@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚5496096 PIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚67 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 UBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚48060432690688 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚41348020 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 123731968 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚321 _runtime‚ 465.153878296 K?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚5865241 PDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚5496165 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 123731968 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚262 K?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚6462393 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 OCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚5869528 PCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚59049984 JBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚665 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚209 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 IBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚59 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 VEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 421953283584 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 4517085184 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 5492441088 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚12331923 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 4628521206 \IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚47638437462016 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚4287 bQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 446337356288 _LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚47358844043264 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 831258624 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚41940992 ^MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 446295415296 KDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚59 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 ]LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 421114594264 C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚4289 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 4552982528 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4289 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚59395584 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 `MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚41543896956928 BDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚48060373640704 XEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚47638420357120 KBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚4876 IBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚69 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚1474120 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚20643840 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 123731968 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚321 [HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚47638420357120 XEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚47779958637528‘0`—O dQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚41543917600768 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚321 OCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚6462395 L@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚5496096 PIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚67 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 UBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚48060432690688 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚41348020 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 123731968 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚321 _runtime‚ 465.153878296 K?DeviceStatsMonitor.on_train_batch_start/active.small_pool˜05ä=#.freed‚5865241 PDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚5496165 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 123731968 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚262 K?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚6462393 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 OCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚5869528 PCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚59049984 JBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚665 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚209 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 IBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚59 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 VEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 421953283584 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 4517085184 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 5492441088 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚12331923 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 4628521206 \IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚47638437462016 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚4287 bQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 446337356288 _LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚47358844043264 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 831258624 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚41940992 ^MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 446295415296 KDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚59 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 ]LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 421114594264 C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚4289 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 4552982528 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4289 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚59395584 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 `MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚41543896956928 BDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚48060373640704 XEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚47638420357120 KBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚4876 IBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚69 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚1474120 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚20643840 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 123731968 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚321 [HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚47638420357120 XEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚47779958637528 ]JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚41990254957056 YFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚41990192372224 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 [HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚47358827003904 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17104896 F?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚59 % _timestamp‚1.7866591166207519e+09 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 4654602240 SGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚5869528 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚1474053 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚41945088 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚321 M@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚59049984 TADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚48060373640704 IDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 4552982528 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚223 YHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 421073246244 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚262 @7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚5264 ]LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 421995228672 YHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 421953283584 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 843096576 ZIDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 421995228672 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 122905600 _LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚47638437462016 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 122905600 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 F>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚4022043 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 5368709120 OCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚6462393 UHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚62584832 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 PCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚58387380 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 5492441088 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 5492441088 trainer/global_step‚1749 IDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4876 OCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚5865241 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 TADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚47779900250148 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 _step‚141 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚4287 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚41945088 D;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚5264 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚4022045 SGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚6462395 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚429†ãóÇ0‘+ _wandbruntime‚465  _runtime‚4652“/>¢&’œM A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚4295 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 123731968 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚321 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚4293 R?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚48086927047168 QEDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚6465948 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚1474824 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 5368709120 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚262 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521206 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚12334474 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚20643840 `ODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 446583145984 K>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59053056 XGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 422226300416 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚41948160 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚223 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚12338769 MADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚6465948 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 122905600 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚12334474 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚209 trainer/global_step‚1749 I=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚6465946 S@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚48086986100224 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 4654602240 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654602240 G@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚68 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚321 G@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚59 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17104896 NBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚5499140 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚46132224 \KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 446537013760 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 843096576 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 BDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 QEDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚5872821 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 122306526 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 5492441088 bODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚41566964973568 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 123731968 EDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚5499072 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 4552982528 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚12338769 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚59395584 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 123731968 >5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5264 I@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚4876 ^KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚41566944329728 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 IBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚59 NGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚66 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 ZGDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚47664759799808 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 % _timestamp‚1.7866591167907627e+09 VCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚47664742694912 B9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚5264 NADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚58387428 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 5368709120 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚321 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 122905600 YFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚47385012797440 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚262 TCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 422184352256 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17104896 MADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚6465946 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚1474890 [HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚42013548119552 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 ]JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚47385029836800 D=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚59 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 4552982528 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 831258624 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 _step‚142 DDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 I=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚5868528 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚4293 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚4024248 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 [JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 421345166924(†B$§&“¡M MADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚5868528 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚41348068 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 QEDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚5872821 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 122306526 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 5492441088 bODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚41566964973568 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 123731968 EDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚5499072 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 4552982528 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚12338769 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚59395584 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 123731968 >5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5264 I@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚4876 ^KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚41566944329728 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 IBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚59 NGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚66 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 ZGDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚47664759799808 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 % _timestamp‚1.7866591167907627e+09 VCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚47664742694912 B9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚5264 NADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚58387428 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 5368709120 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚321 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 122905600 YFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚47385012797440 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚262 TCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 422184352256 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17104896 MADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚6465946 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚1474890 [HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚42013548119552 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 ]JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚47385029836800 D=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚59 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 4552982528 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 831258624 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 _step‚142 DDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 I=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚5868528 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚4293 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚4024248 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 [JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 421345166924 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚4295 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 123731968 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚321 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚4293 R?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚48086927047168 QEDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚6465948 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚1474824 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 5368709120 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚262 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521206 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚12334474 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚20643840 `ODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 446583145984 K>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59053056 XGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 422226300416 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚41948160 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚223 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚12338769 MADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚6465948 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 122905600 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚12334474 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚209 trainer/global_step‚1749 I=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚6465946 S@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚48086986100224 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 4654602240 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654602240 G@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚68 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚321 G@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚59 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17104896 NBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚5499140 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚46132224 \KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 446537013760 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 843096576 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 Bv–ð trainer/global_step‚1749 % _timestamp‚1.7866591167919407e+09 epoch‚4 _runtime‚ 465.321824779 _step‚143 0loss/FRt1/assignment_loss‚0.9506264259418269 0loss/FRt2/assignment_loss‚0.8867463982221447 /loss/FRt1/detection_loss‚0.5682185801397621 /loss/FRt2/detection_loss‚0.5295737732862005 &loss/total_loss‚2.9351651775899343Òˆ—þ0—+ _wandbruntime‚479  _runtime‚4791ärÓ˜¼ % lr-AdamW/pg1‚0.001481222674876112 % lr-AdamW/pg2‚0.001481222674876112 trainer/global_step‚1799 $ _timestamp‚1.786659130608488e+09 _runtime‚ 479.13854821 _step‚144ͱ¦Ç™Á % lr-AdamW/pg2‚0.001481222674876112 trainer/global_step‚1799 $ _timestamp‚1.786659130608488e+09 _runtime‚ 479.13854821 _step‚144 % lr-AdamW/pg1‚0.001481222674876112ÈÿQ0š+ _wandbruntime‚479  _runtime‚479h ²-ö ›’O QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 5492441088 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚321 Q>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚49388077296128 PCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚59049984 SGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚6034630 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚223 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚1514944 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚41348020 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚262 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚262 \IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚48954554351616 `MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚42696265596928 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 K?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚6640043 ]LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 433581994496 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 122306526 D;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚5264 ^MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 458568449536 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 5492441088 % _timestamp‚1.7866591306094825e+09 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 123731968 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 4654602240 YHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 433540049408 L@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚5647165 BDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4876 SGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚6640045 OCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚6640043 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 F?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚59 C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚4289 IBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚59 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚429 UHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚68876288 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 5368709120 dQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚42696286240768 PCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚58387380 _LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚48668133720064 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 4517085184 OCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚6034630 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4289 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚4287 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17104896 [HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚48954537246720 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 ]LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 432676404872 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 123731968 ZIDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 433581994496 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 4654602240 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚4287 XEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚49100810124936 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 5492441088 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚262 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 5368709120 _runtime‚ 479.142876517 @7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚5264 TADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚49100751737556 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 4552982528 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 122905600 JBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚665 OCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚6030343 IBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚74 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚4132295 YFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚43154834046464 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚41945088 IDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 4552982528 ]JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚43154902922752 bQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 458616681984 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 4628521206 XEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚48954537246720 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 PDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚5647239 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚20643840 YHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 432635056852 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 5368709120 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚321 UBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚49388136346112 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 831258624 _step‚145 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 122905600 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚321 KDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚59 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚41945088 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚4132293 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0ó"Râ'œ—O JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚4287 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17104896 [HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚48954537246720 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 ]LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 432676404872 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 123731968 ZIDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 433581994496 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 4654602240 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚4287 XEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚49100810124936 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 5492441088 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚262 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 5368709120 _runtime‚ 479.142876517 @7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚5264 TADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚49100751737556 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 4552982528 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 122905600 JBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚665 OCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚6030343 IBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚74 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚4132295 YFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚43154834046464 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚41945088 IDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 4552982528 ]JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚43154902922752 bQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 458616681984 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 4628521206 XEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚48954537246720 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 PDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚5647239 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚20643840 YHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 432635056852 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 5368709120 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚321 UBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚49388136346112 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 831258624 _step‚145 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 122905600 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚321 KDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚59 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚41945088 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚4132293 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 5492441088 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚321 Q>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚49388077296128 PCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚59049984 SGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚6034630 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚223 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚1514944 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚41348020 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚262 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚262 \IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚48954554351616 `MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚42696265596928 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 K?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚6640043 ]LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 433581994496 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 122306526 D;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚5264 ^MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 458568449536 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 5492441088 % _timestamp‚1.7866591306094825e+09 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 123731968 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 4654602240 YHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 433540049408 L@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚5647165 BDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4876 SGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚6640045 OCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚6640043 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 F?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚59 C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚4289 IBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚59 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚429 UHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚68876288 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 5368709120 dQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚42696286240768 PCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚58387380 _LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚48668133720064 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 4517085184 OCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚6034630 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4289‘²ÉOÑ0+ _wandbruntime‚479  _runtime‚479&ë½ &žšM TCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 433771118080 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 5368709120 BDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 4654602240 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17104896 WFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 433771118080 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚1515703 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 5368709120 ODeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654602240 ^KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚42719312969728 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 ]JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚48694319513600 [HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚43178191981056 E5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5264 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚321 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 D=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚59 trainer/global_step‚1799 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚41948160 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚12677226 MADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚6643596 NGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚69 SFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚64678912 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 123731968 I=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚6033630 K>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59053056 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 4552982528 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚12681521 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚321 G@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚59 I=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚6643596 MADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚6037923 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 122905600 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚209 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 831258624 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚44035072 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 WFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 432865629464 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚12677226 NADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚59053056 H@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚665 WDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚43178127302144 bODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚42719333613568 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚262 [JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 432906977532 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17104896 VCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚48980859584512 [JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 433813066240 QEDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚6037923 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚4134498 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚429 XGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 433813066240 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚20643840 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚59395584 YFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚48694302474240 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚321 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚262 ZGDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚48980876689408 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 ]JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚48980876689408 G@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚71 VCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚49414689755648 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 843096576 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚262 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 R?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚49127168103704 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚4295 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 4552982528 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 122905600 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 $ _timestamp‚1.786659130783274e+09 DDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚5650201 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚223 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521206 MADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚6033630 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 5492441088 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 4517085184 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 5492441088 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 123731968 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 123731968 IBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚59Ç›ÎΟŸM >5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5264 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚321 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 D=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚59 trainer/global_step‚1799 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚41948160 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚12677226 MADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚6643596 NGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚69 SFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚64678912 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 123731968 I=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚6033630 K>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59053056 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 4552982528 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚12681521 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚321 G@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚59 I=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚6643596 MADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚6037923 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 122905600 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚209 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 831258624 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚44035072 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 WFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 432865629464 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚12677226 NADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚59053056 H@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚665 WDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚43178127302144 bODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚42719333613568 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚262 [JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 432906977532 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17104896 VCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚48980859584512 [JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 433813066240 QEDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚6037923 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚4134498 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚429 XGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 433813066240 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚20643840 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚59395584 YFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚48694302474240 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚321 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚262 ZGDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚48980876689408 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 ]JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚48980876689408 G@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚71 VCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚49414689755648 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 843096576 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚262 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 R?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚49127168103704 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚4295 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 4552982528 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 122905600 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 F@DeviceStatsMonitor.on_train_batch_end/oversize_alÒ~j¿×locations.freed‚0 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 $ _timestamp‚1.786659130783274e+09 DDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚5650201 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚223 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521206 MADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚6033630 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 5492441088 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 4517085184 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 5492441088 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 123731968 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 123731968 IBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚59 TCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 433771118080 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 5368709120 BDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 4654602240 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17104896 WFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 433771118080 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚1515703 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 5368709120 ODeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654602240 ^KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚42719312969728 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 ]JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚48694319513600 [HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚43178191981056 EDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4876 UBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚50715840001536 YFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚44319466010624 F?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚59 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚262 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 Q>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚50715780951552 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚41945088 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 831258624 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 IBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚59 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚4287 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚13017427 SGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚6817695 OCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚6195445 PDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚5798020 bQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 470871617536 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚209 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚59395584 PCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚59049984 _LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚50270671241216 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 TADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚50421603224964 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 5368709120 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 843096576 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 trainer/global_step‚1849 K?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚6817693 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17104896 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 4654602240 ]LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 445168760320 _LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚49977423396864 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 5492441088 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 4517085184 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 F>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 OCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚6199732 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 123731968 _step‚149 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 4654602240 OCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚6817693 SGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚6199732 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚4242543 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 5368709120 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 4552982528 VEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 445126815232 M@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚59049984 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚41945088 TADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚50715780951552 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 @7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚5264 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17104896 JBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚665 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚1555475 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 [HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚50270654136320 IBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚62 [HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚49977406357504 % _timestamp‚1.7866591434351962e+09 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚262 OCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚6817695 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 122905600 D;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚5264 PIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚60 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 123731968 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚262 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 4628521206 \IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚50270671241216 L@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚5797958 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚321 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 4552982528 ^MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 470831773696 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 122905600 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4289 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚4242545 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚223 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 5492441088 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚321 ZIDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 445168760320 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚4287 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚39843840 XEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚50421661612344 E8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚13013138 XEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚50270654136320 KBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚4876 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚429 ]JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚44319526498304 BDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4876 UBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚50715840001536 YFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚44319466010624 F?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚59 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚262 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 Q>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚50715780951552 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚41945088 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 831258624 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 IBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚59 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚4287 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚13017427 SGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚6817695 OCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚6195445 PDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚5798020 bQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 470871617536 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚209 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚59395584 PCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚59049984 _LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚50270671241216 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 TADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚50421603224964 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 5368709120 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 843096576 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 trainer/global_step‚1849 K?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚6817693 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17104896 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 4654602240 ]LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 445168760320 _LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚49977423396864 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 5492441088 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 4517085184 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 F>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 OCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚6199732 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 123731968 _step‚149 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 4654602240 OCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚6817693 SGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚6199732 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚4242543 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 5368709120 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 4552982528 VEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 445126815232 M@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚59049984 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚41945088 TADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚50715780951552 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 @7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚5264 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17104896 JBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚665 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚1555475 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 [HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚50270654136320 IBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚62 [HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚49977406357504 % _timestamp‚1.7866591434351962e+09 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚262 OCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚6817695 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 122905600 D;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚5264 PIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚60 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 123731968 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚262 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 4628521206 \IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚50270671241216 L@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚5797958 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚321 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 4552982528 ^MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 470831773696 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 122905600 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4289 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚4242545 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚223 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 5492441088 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚321 ZIDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 445168760320 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚4287 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚39843840 XEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚50421661612344 E8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚13013138 XEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚50270654136320 KBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚4876 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚429 ]JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚44319526498304 B6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚321 S@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚50742393411072 TCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 445357883904 J>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚5800968 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 YFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚50003592151040 \KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 471071889408 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 5368709120 E5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5264 ODeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654602240 NGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚64 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚48229376 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 DDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 % _timestamp‚1.7866591436098886e+09 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 4552982528 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚1556284 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 ]JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚50296993579008 ^KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚43871681609728 [JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 444468788140 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚4295 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17104896 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 5492441088 VCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚50296976474112 WFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 445357883904 I@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚4876 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚59395584 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 VCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚50448077978540 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚41348068 VCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚50742393411072 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚429 bODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚43871702253568 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521206 SEDeviceStaH ÎftsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 122306526 D=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚59 trainer/global_step‚1849 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚13024273 [JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 445399832064 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 122905600 EDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚13024273 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚321 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 R?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚50448019591112 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 123731968 MADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚6203025 YFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚50296976474112 H@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚665 [HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚44342822372352 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 4517085184 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 123731968 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 4552982528 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚1556220 XGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 445399832064 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚41948160 NBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚5801034 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 831258624 _step‚150 R?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚50742334358016 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚4293 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚20643840 `ODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 471120118784 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 NADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚58387428 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚321 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 5492441088 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 5368709120 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 MADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚6198732 _runtime‚ 492.140375405 MADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚6821248 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 122905600 NADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚59053056 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17104896 WDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚44342753499136 K>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59053056 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚13019978 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 IBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚59 ZGDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚50296993579008 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 4654602240 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚4244750 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 QEDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚6203025 SFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚68873216 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 843096576 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚262 G@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚59 I=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚6821246 B9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚5264 WFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 444427440072–ÇÕ §&­¡M MADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚6198732 _runtime‚ 492.140375405 MADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚6821248 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 122905600 NADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚59053056 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17104896 WDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚44342753499136 K>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59053056 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚13019978 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 IBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚59 ZGDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚50296993579008 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 4654602240 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚4244750 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 QEDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚6203025 SFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚68873216 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 843096576 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚262 G@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚59 I=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚6821246 B9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚5264 WFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 444427440072 MADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚6821246 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚13019978 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 5368709120 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚321 S@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚50742393411072 TCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 445357883904 J>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚5800968 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 YFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚50003592151040 \KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 471071889408 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 5368709120 E5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5264 ODeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654602240 NGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚64 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚48229376 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 DDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 % _timestamp‚1.7866591436098886e+09 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 4552982528 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚1556284 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 ]JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚50296993579008 ^KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚43871681609728 [JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 444468788140 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚4295 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17104896 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 5492441088 VCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚50296976474112 WFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 445357883904 I@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚4876 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚59395584 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 VCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚50448077978540 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚41348068 VCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚50742393411072 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚429 bODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚43871702253568 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521206 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 122306526 D=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚59 trainer/global_step‚1849 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚13024273 [JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 445399832064 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 122905600 EDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚13024273 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚321 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 R?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚50448019591112 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 123731968 MADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚6203025 YFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚50296976474112 H@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚665 [HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚44342822372352 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 4517085184 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 123731968 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 4552982528 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚1556220 XGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 445399832064 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚41948160 NBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚5801034 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 831258624 _step‚150 R?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚50742334358016 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚4293 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚20643840 `ODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 471120118784 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 NADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚58387428 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚321 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 5492441088 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 5368709120 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0–o ô³0®+ _wandbruntime‚492  _runtime‚492b©Š—q¯ë % _timestamp‚1.7866591436112604e+09 0loss/FRt2/assignment_loss‚0.9316242912199171 &loss/total_loss‚3.0379812554399006 epoch‚4 trainer/global_step‚1849 _runtime‚ 492.141199491 _step‚151 0loss/FRt1/assignment_loss‚1.0043684112539597 /loss/FRt1/detection_loss‚0.5404606094794084 /loss/FRt2/detection_loss‚0.5615279434866152DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚52043484606976 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚13360179 XEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚51586771025920 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚41945088 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚1596191 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 ]LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 455800026088 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 123731968 ZIDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 456755526144 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 122905600 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚41945088 UHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚64681984 BDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 TADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚52043484606976 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 5368709120 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 4654602240 @7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚5264 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 4654602240 JBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚665 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚262 OCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚6995345 OCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚6364834 dQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚45001023520768 `MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚45001002876928 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 OCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚6995343 ]JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚45484197458432 _LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚51286713073664 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 \IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚51586788130816 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 4517085184 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 5492441088 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 UBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚52043543656960 _LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚51586788130816 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 123731968 VEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 456713581056 XEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚51742513099752 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 5368709120 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚262 bQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 483173937664 D;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚5264 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚20643840 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 122306526 K?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚6995343 YHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 456713581056 IBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚73 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚4352793 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚209 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 IBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚59 G>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4876 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚321 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 [HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚51286696034304 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚4287 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚223 YFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚45484132776448 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 4552982528 IDeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4289 SGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚6995345 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚1596262 PIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚71 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚321 KDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚59 SGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚6364834 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 122905600 KBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚4876 PDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚5949057 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0|²ŽfÛ¸–O UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 122905600 KBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚4876 PDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚5949057 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 IDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚52043484606976 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚13360179 XEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚51586771025920 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚41945088 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚1596191 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 ]LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 455800026088 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 123731968 ZIDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 456755526144 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 122905600 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚41945088 UHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚64681984 BDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 TADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚52043484606976 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 5368709120 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 4654602240 @7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚5264 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 4654602240 JBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚665 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚262 OCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚6995345 OCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚6364834 dQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚45001023520768 `MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚45001002876928 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 OCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚6995343 ]JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚45484197458432 _LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚51286713073664 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 \IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚51586788130816 VGDeviceStatsMonitor.on_train_ba·’òÁtch_start/requested_bytes.large_pool.peak‚ 4517085184 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 5492441088 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 UBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚52043543656960 _LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚51586788130816 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 123731968 VEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 456713581056 XEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚51742513099752 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 5368709120 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚262 bQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 483173937664 D;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚5264 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚20643840 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 122306526 K?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚6995343 YHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 456713581056 IBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚73 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚4352793 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚209 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 IBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚59 G>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4876 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚321 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 [HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚51286696034304 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚4287 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚223 YFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚45484132776448 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 4552982528 IDeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4289 SGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚6995345 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚1596262 PIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚71 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚321 KDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚59 SGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚6364834 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0™ü5ó40¹+ _wandbruntime‚505  _runtime‚505á°2ý¢&ºœM [HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚45507491595776 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 843096576 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚4293 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 122905600 EDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 VCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚51613093363712 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚13362730 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 QEDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚6998898 H@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚665 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚4355000 WDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚45507422722560 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 ZGDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚51613110468608 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 122905600 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚1597069 NGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚70 bODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚45024070893568 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 831258624 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 4517085184 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 5368709120 S@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚52070097066496 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 5368709120 D=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚59 VCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚51768929465948 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚262 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚13367025 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚4295 NADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚59053056 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚321 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 4654602240 NBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚5952069 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚209 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚321 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 BDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚5951997 R?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚52070038013440 MADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚6998896 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 YFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚51312881827840 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚429 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚223 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 WFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 455989250680 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚321 I=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚6998896 DDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚13367025 G@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚72 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 122306526 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 5368709120 MADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚6368127 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17104896 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚59395584 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 EDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 VCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚52070097066496 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17104896 ^KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚45024050249728 _runtime‚ 505.189907466 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚13362730 >5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5264 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚20643840 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521206 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654602240 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚4354998 G@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚59 K>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59053056 [JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 456986597888 MADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚6363834 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚41948160 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚4293 % _timestamp‚1.7866591566593444e+09 XGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 456986597888 I@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚4876 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚48229376 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 MADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚6998898 B9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚5264m?$æ§&»¡M MADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚6368127 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17104896 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚59395584 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 EDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 VCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚52070097066496 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17104896 ^KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚45024050249728 _runtime‚ 505.189907466 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚13362730 >5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5264 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚20643840 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521206 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654602240 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚4354998 G@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚59 K>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59053056 [JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 456986597888 MADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚6363834 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚41948160 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚4293 % _timestamp‚1.7866591566593444e+09 XGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 456986597888 I@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚4876 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚48229376 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 MADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚6998898 B9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚5264 [HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚45507491595776 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 843096576 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚4293 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 122905600 EDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 VCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚51613093363712 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚13362730 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 QEDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚6998898 H@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚665 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚4355000 WDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚45507422722560 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 ZGDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚51613110468608 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 122905600 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚1597069 NGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚70 bODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚45024070893568 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 831258624 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 4517085184 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 5368709120 S@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚52070097066496 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 5368709120 D=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚59 VCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚51768929465948 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚262 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚13367025 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚4295 NADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚59053056 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚321 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 4654602240 NBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚5952069 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚209 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚321 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 BDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚5951997 R?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚52070038013440 MADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚6998896 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 YFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚51312881827840 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚429 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚223 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 WFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 455989250680 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚321 I=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚6998896 DDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚13367025 G@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚72 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 122306526 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 5368709120šL#0¼+ _wandbruntime‚505  _runtime‚505ëæo½é /loss/FRt1/assignment_loss‚0.815270128362844 0loss/FRt2/assignment_loss‚0.8303716105595454 /loss/FRt1/detection_loss‚0.5382298665110018 epoch‚4 % _timestamp‚1.7866591566605318e+09 _runtime‚ 505.190370191 _step‚155 /loss/FRt2/detection_loss‚0.5183609602399668 %loss/total_loss‚2.702232565673358 trainer/global_step‚1899ŸÊjkt¾î /loss/FRt1/assignment_loss‚0.815270128362844 0loss/FRt2/assignment_loss‚0.8303716105595454 /loss/FRt1/detection_loss‚0.5382298665110018 epoch‚4 % _timestamp‚1.7866591566605318e+09 _runtime‚ 505.190370191 _step‚155 /loss/FRt2/detection_loss‚0.5183609602399668 %loss/total_loss‚2.702232565673358 trainer/global_step‚1899›b )ž–¿‚(:‹ Ù‚ùÓº½êá gpu.0.gpu‚63.0 gpu.0.memory‚12, gpu.0.memoryAllocated‚7.0288848876953125) gpu.0.memoryAllocatedBytes‚ 6037766144 gpu.0.temp‚36.0 gpu.0.powerWatts‚129.798' gpu.0.enforcedPowerLimitWatts‚500.0 gpu.0.powerPercent‚25.9596 gpu.0.smClock‚1410 gpu.0.memoryClock‚1593$ gpu.0.correctedMemoryErrors‚1955# gpu.0.uncorrectedMemoryErrors‚0š’8ë†À‚(:û Ù‚ùÓŸÖÈå disk.sda3.in‚ 4785.328125 disk.sda2.out‚0 disk.dm-1.in‚ 4785.328125 disk.drbd2165.in‚0 disk.drbd2165.out‚0 disk.dm-0.in‚ 157.17578125* proc.memory.percent‚14.504456520080566" disk.nvme0c0n1.in‚ 157.17578125 disk.sda.out‚9.765625 disk.sda1.out‚0 disk.sda2.in‚0" disk.dm-0.out‚3371.91357421875$ memory_percent‚64.86871242523193 network.recv‚ 1.071567e+06$ disk./.usageGB‚441.8393249511719 disk.sda3.out‚9.765625' disk.nvme0c0n1.out‚3371.91357421875 disk.sda.in‚ 4785.328125 disk.sda1.in‚0% disk.nvme0n1.out‚3371.91357421875) proc.memory.availableMB‚ 5755.91015625 network.sent‚ 2.52941e+06) disk./.usagePercent‚29.65220782535977 disk.nvme0n1.in‚ 157.17578125 disk.dm-1.out‚9.765625# proc.memory.rssMB‚ 2376.41015625 cpu‚26.418834336658886 proc.cpu.threads‚17çPçÈ0Á+ _wandbruntime‚517  _runtime‚517NKªÁÆÂÀ _runtime‚ 517.340120629 _step‚156 & lr-AdamW/pg1‚0.0014770020254035617 & lr-AdamW/pg2‚0.0014770020254035617 trainer/global_step‚1949 % _timestamp‚1.7866591688100517e+09š½¹LËÃÅ & lr-AdamW/pg2‚0.0014770020254035617 trainer/global_step‚1949 % _timestamp‚1.7866591688100517e+09 _runtime‚ 517.340120629 _step‚156 & lr-AdamW/pg1‚0.0014770020254035617œWp ß0Ä+ _wandbruntime‚517  _runtime‚5172,µDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4876 OCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚7172995 XEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚52902887915520 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚20643840 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚4289 _LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚52902905020416 YFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚46648796622336 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚321 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 122306526 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚262 F>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 PDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚6100008 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 TADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚53063306199780 % _timestamp‚1.7866591688110797e+09 E8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚13698642 UHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚73070592 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 5368709120 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 122905600 IDeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4289 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 831258624 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 [HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚52595985711104 Q>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚53371188262400 dQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚46153392160768 XEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚53063364587160 trainer/global_step‚1949 JBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚665 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 `MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚46153371516928 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚321 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 IBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚59 IBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚86 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚262 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚13702931 D;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚5264 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 5492441088 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 123731968 _step‚157 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚4287 PIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚84 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 PCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚58387380 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 123731968 PCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚59049984 TADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚53371188262400 [HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚52902887915520 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 122905600 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚4287 bQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 495477532160 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 ]JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚46648869692928 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 5492441088 M@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚59049984 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17104896 SGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚6529936 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚1636963 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚209‡m¡ì'Æ—O TADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚53063306199780 % _timestamp‚1.7866591688110797e+09 E8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚13698642 UHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚73070592 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 5368709120 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 122905600 IDeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4289 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 831258624 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 [HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚52595985711104 Q>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚53371188262400 dQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚46153392160768 XEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚53063364587160 trainer/global_step‚1949 JBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚665 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 `MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚46153371516928 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚321 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 IBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚59 IBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚86 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚262 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚13702931 D;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚5264 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 5492441088 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 123731968 _step‚157 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚4287 PIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚84 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 PCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚58387380 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 123731968 PCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚59049984 TADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚53371188262400 [HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚52902887915520 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 122905600 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚4287 bQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 495477532160 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 ]JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚46648869692928 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 5492441088 M@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚59049984 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17104896 SGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚6529936 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚1636963 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚209 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚262 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 5368709120 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚41945088 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 4654602240 IDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4876 OCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚7172995 XEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚52902887915520 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚20643840 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚4289 _LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚52902905020416 YFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚46648796622336 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚321 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 122306526 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚262 F>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 PDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚6100008 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0øm€d0Ç+ _wandbruntime‚517  _runtime‚517_GÌt¡&È›M MADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚6533229 B9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚5264 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 4654602240 \KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 495670988288 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚13709777 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚20643840 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 I=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚7176546 QEDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚6533229 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚1637764 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 5368709120 S@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚53397800721920 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 122905600 [HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚46672162945536 I=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚6528936 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17104896 SFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚73067520 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 843096576 R?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚53397741668864 QEDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚7176548 I@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚4876 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521206 YFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚52622171504640 G@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚59 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚4293 TCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 468531415552 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚13705482 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 831258624 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 J>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚6102943 ]JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚52622188544000 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚429 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚13705482 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 WDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚46672089878016 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 5368709120 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 4552982528 H@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚665 MADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚6528936 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 WFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 467551061288 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚52423680 B5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5264 `ODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 495723411968 NADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚58387428 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚321 D=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚59 [JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 468573363712 IBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚59 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 4552982528 ^KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚46176418889728 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 5368709120 NGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚69 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚1637695 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 123731968 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚41348068 K>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59053056 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17104896 XGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 468573363712 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 122905600 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 _runtime‚ 517.502277569 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚13709777 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚41948160 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚321 [JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 467592409356 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 5492441088 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 5492441088 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 123731968 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 _step‚158 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚209 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 122306526 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 5492441088 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 trainer/global_step‚1949 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚4295 YFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚52929210253312 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚41948160 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚59395584 VCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚53089780953356 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚4465248 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚321 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚262 EDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚4465250 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654602240 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 123731968 VCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚53397800721920 WFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 468531415552 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚223 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 4517085184 ZGDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚52929227358208 NBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚6103014 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚321 DDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚4465250 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654602240 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 123731968 VCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚53397800721920 WFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 468531415552 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚223 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 4517085184 ZGDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚52929227358208 NBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚6103014 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚321 DDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚13709777 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚20643840 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 I=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚7176546 QEDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚6533229 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚1637764 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 5368709120 S@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚53397800721920 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 122905600 [HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚46672162945536 I=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚6528936 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17104896 SFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚73067520 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 843096576 R?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚53397741668864 QEDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚7176548 I@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚4876 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521206 YFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚52622171504640 G@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚59 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚4293 TCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 468531415552 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚13705482 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 831258624 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 J>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚6102943 ]JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚52622188544000 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚429 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚13705482 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 WDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚46672089878016 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 5368709120 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 4552982528 H@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚665 MADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚6528936 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 WFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 467551061288 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚52423680 B5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5264 `ODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 495723411968 NADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚58387428 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚321 D=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚59 [JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 468573363712 IBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚59 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 4552982528 ^KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚46176418889728 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 5368709120 NGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚69 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚1637695 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 123731968 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚41348068 K>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59053056 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17104896 XGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 468573363712 SÓ’ã/‹EDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 122905600 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 _runtime‚ 517.502277569 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚13709777 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚41948160 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚321 [JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 467592409356 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 5492441088 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 5492441088 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 123731968 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 _step‚158 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚209 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 122306526 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 5492441088 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 trainer/global_step‚1949 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚4295 YFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚52929210253312 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚41948160 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚59395584 VCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚53089780953356 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚4465248 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚321 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚262 EDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚54698891917824 XEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚54698950967808 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚4573293 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚321 F>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 OCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚6690751 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 122905600 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 122306526 XEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚54219004805120 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4289 YFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚47813432676352 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 G>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4876 PDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚6250884 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 _step‚161 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚4287 D;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚5264 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚20643840 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚59395584 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 bQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 507734460416 ^MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 507692519424 K?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚6690751 L@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚6250816 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 4517085184 UBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚54698950967808 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 4552982528 _LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚53905292427264 KDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚59 OCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚7350645 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚4287 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚1677523 `MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚47305740156928 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚321 BDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 YHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 478882299284 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 M@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚59049984 SGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚6695038 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 ]LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 478923647304 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 TADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚54698891917824 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 4654602240 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚429 dQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚47305760800768 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 123731968 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚321 PCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚59049984 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 5368709120 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚262 IDeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4289 YFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚47813432676352 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 G>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4876 PDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚6250884 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 _step‚161 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚4287 D;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚5264 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚20643840 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚59395584 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 bQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 507734460416 ^MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 507692519424 K?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚6690751 L@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚6250816 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 4517085184 UBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚54698950967808 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 4552982528 _LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚53905292427264 KDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚59 OCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚7350645 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚4287 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚1677523 `MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚47305740156928 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚321 BDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 YHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 478882299284 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 M@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚59049984 SGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚6695038 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 ]LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 478923647304 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 TADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚54698891917824 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 4654602240 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚429 dQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚47305760800768 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 123731968 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚321 PCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚59049984 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 5368709120 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚262 IDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚54698891917824 XEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚54698950967808 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚4573293 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚321 F>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 OCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚6690751 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 122905600 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 122306526 XEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚54219004805120¡¿÷`0Õ+ _wandbruntime‚529  _runtime‚5290­À„ÖœM [HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚47836786940928 BDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚14052529 MADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚7354196 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚1678317 VCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚54410632440764 DDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚6253815 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 SFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚62581760 bODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚47328808173568 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 5492441088 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 122905600 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚1678373 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 trainer/global_step‚1999 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚223 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚41948160 XGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 480160129536 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521206 MADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚6698331 K>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59053056 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 843096576 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 4552982528 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚262 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 I@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚4876 R?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚54410574053336 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 5368709120 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚4293 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚321 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 _runtime‚ 529.473021166 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚41937920 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚14052529 R?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚54725445324288 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚41948160 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚4575498 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 122306526 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚262 E6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚321 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚4295 VCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚54725504377344 YFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚54245327142912 [JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 480160129536 NADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚58387428 MADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚7354198 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654602240 YFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚53931461181440 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 5368709120 E5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5264 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚14048234 QEDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚7354198 [JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 479154219964 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 G@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚58 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 123731968 H@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚665 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚59395584 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 ODeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚14052529 MADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚7354196 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚1678317 VCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚54410632440764 DDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚6253815 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 SFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚62581760 bODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚47328808173568 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 5492441088 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 122905600 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚1678373 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 trainer/global_step‚1999 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚223 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚41948160 XGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 480160129536 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521206 MADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚6698331 K>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59053056 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 843096576 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 4552982528 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚262 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 I@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚4876 R?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚54410574053336 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 5368709120 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚4293 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚321 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 _runtime‚ 529.473021166 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚41937920 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚14052529 R?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚54725445324288 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚41948160 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚4575498 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 122306526 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚262 E6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚321 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚4295 VCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚54725504377344 YFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚54245327142912 [JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 480160129536 NADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚58387428 MADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚7354198 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654602240 YFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚53931461181440 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 5368709120 E5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5264 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚14048234 QEDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚7354198 [JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 479154219964 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 G@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚58 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 123731968 H@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚665 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚59395584 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 ODeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 PCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚59049984 BDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚41348020 C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚4289 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 4654602240 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4289 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚4287 YFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚49281465966080 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 _LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚55595481595904 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 123731968 PDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚6451754 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚223 [HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚55595464556544 KDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚59 IBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚59 G>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4876 Q>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚56421600514560 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 ]LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 495583549952 [HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚55926058909696 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 XEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚56090033025922 VEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 495541604864 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 843096576 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 5492441088 trainer/global_step‚2049 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 4628521206 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 123731968 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚321 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚262 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 122905600 D;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚5264 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚1727595 `MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚48757379104768 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 5368709120 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚321 ZIDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 495583549952 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 5492441088 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚321 UBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚56421659564544 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 OCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚6882332 JBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚665 E8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚14466944 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 K?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚7588899 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚1727536 UHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚66779136 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 \IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚55926076014592 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚41945088 SGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚6882332 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 _runtime‚ 547.164133665 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17104896 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚14471233 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚4724159 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚262 M@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚59049984 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 4552982528 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 5492441088 IDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 PCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚59049984 BDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚41348020 C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚4289 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 4654602240 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4289 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚4287 YFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚49281465966080 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 _LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚55595481595904 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 123731968 PDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚6451754 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚223 [HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚55595464556544 KDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚59 IBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚59 G>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4876 Q>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚56421600514560 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 ]LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 495583549952 [HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚55926058909696 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 XEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allË"(…yocated‚56090033025922 VEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 495541604864 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 843096576 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 5492441088 trainer/global_step‚2049 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 4628521206 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 123731968 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚321 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚262 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 122905600 D;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚5264 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚1727595 `MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚48757379104768 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 5368709120 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚321 ZIDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 495583549952 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 5492441088 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚321 UBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚56421659564544 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 OCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚6882332 JBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚665 E8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚14466944 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 K?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚7588899 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚1727536 UHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚66779136 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 \IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚55926076014592 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚41945088 SGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚6882332 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 _runtime‚ 547.164133665 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17104896 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚14471233 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚4724159 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚262 M@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚59049984 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 4552982528 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 5492441088 IDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚321 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚14478079 K>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59053056 J>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚6454693 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚209 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 122905600 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17104896 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚14473784 MADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚6885625 MADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚7592452 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚41348068 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 123731968 YFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚55952381247488 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚14473784 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚4726364 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚1728331 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 5368709120 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚262 D=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚59 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 WFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 494740654610 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 5492441088 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 123731968 >5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5264 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 4654602240 TCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 495772673536 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚59395584 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚321 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 5368709120 QEDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚6885625 ^KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚48780426477568 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚262 VCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚56448212974080 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 G@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚59 G@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚76 [JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 495814621696 B9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚5264 NGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚74 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521206 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 123731968 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 R?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚56116391004690 % _timestamp‚1.7866591987993891e+09 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 4517085184 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 EDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚4295 WFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 495772673536 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚14478079 I=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚7592452 O6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚321 VCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚55952381247488 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚429 WDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚49304757124608 BDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654602240 I@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚4876 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2— ó[§&è¡M =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 R?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚56116391004690 % _timestamp‚1.7866591987993891e+09 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 4517085184 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 EDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚4295 WFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 495772673536 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚14478079 I=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚7592452 O6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚321 VCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚55952381247488 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚429 WDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚49304757124608 BDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654602240 I@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚4876 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 `ODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 524376779264 \KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 524330647040 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 VCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚56116449392118 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚321 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚14478079 K>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59053056 J>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚6454693 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚209 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 122905600 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17104896 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚14473784 MADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚6885625 MADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚7592452 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚41348068 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 123731968 YFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚55952381247488 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚14473784 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚4726364 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚1728331 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 5368709120 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚262 D=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚59 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 WFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 494740654610 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 5492441088 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 123731968 >5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5264 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 4654602240 TCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 495772673536 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚59395584 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚321 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 5368709120 QEDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚6885625 ^KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚48780426477568 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚262 VCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚56448212974080 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 G@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚59 G@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚76 [JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 495814621696 B9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚5264 NGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚74 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521206 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 123731968§=q{!0é+ _wandbruntime‚547  _runtime‚547ºÈw$lêæ /loss/FRt2/assignment_loss‚1.102276645304125 .loss/FRt2/detection_loss‚0.568806064784315 %loss/total_loss‚3.515850169203474 epoch‚5 trainer/global_step‚2049 _runtime‚ 547.3302947 _step‚168 /loss/FRt1/detection_loss‚0.6038220427726275 % _timestamp‚1.7866591988004792e+09 0loss/FRt1/assignment_loss‚1.2409454163424065L£Xqëë _runtime‚ 547.3302947 _step‚168 /loss/FRt1/detection_loss‚0.6038220427726275 % _timestamp‚1.7866591988004792e+09 0loss/FRt1/assignment_loss‚1.2409454163424065 /loss/FRt2/assignment_loss‚1.102276645304125 .loss/FRt2/detection_loss‚0.568806064784315 %loss/total_loss‚3.515850169203474 epoch‚5 trainer/global_step‚2049¨>ßÉK ì‚(:• †ƒùÓÙ¼öà gpu.0.gpu‚40.0 gpu.0.memory‚9, gpu.0.memoryAllocated‚7.0288848876953125) gpu.0.memoryAllocatedBytes‚ 6037766144 gpu.0.temp‚35.0 gpu.0.powerWatts‚204.398' gpu.0.enforcedPowerLimitWatts‚500.0) gpu.0.powerPercent‚40.879599999999996 gpu.0.smClock‚1410 gpu.0.memoryClock‚1593$ gpu.0.correctedMemoryErrors‚1955# gpu.0.uncorrectedMemoryErrors‚0±¯ç…í‚(:ú †ƒùÓó€Ûã network.sent‚ 2.750498e+06$ disk./.usageGB‚441.8469924926758 disk.sda2.in‚0 disk.sda2.out‚0 disk.sda3.in‚ 4785.328125' disk.nvme0c0n1.out‚3450.26025390625 network.recv‚ 1.16451e+06 disk.dm-1.in‚ 4785.328125 disk.sda.out‚ 10.81640625 disk.sda1.in‚0 disk.sda1.out‚0$ memory_percent‚64.82560634613037) proc.memory.percent‚14.45918083190918 cpu‚26.513845932082027 disk.nvme0n1.in‚ 157.28125% disk.nvme0n1.out‚3450.26025390625 disk.dm-1.out‚ 10.81640625 disk.drbd2165.out‚0 disk.sda3.out‚ 10.81640625 disk.sda.in‚ 4785.328125 proc.cpu.threads‚17* disk./.usagePercent‚29.652722400503595 disk.drbd2165.in‚0 disk.nvme0c0n1.in‚ 157.28125) proc.memory.availableMB‚ 5762.97265625" proc.memory.rssMB‚ 2368.9921875 disk.dm-0.in‚ 157.28125" disk.dm-0.out‚3450.26025390625¡MÇÈ0î+ _wandbruntime‚560  _runtime‚560Åšø:ÆïÀ & lr-AdamW/pg1‚0.0014723611480910233 & lr-AdamW/pg2‚0.0014723611480910233 trainer/global_step‚2099 % _timestamp‚1.7866592117080193e+09 _runtime‚ 560.238088355 _step‚169? JÀËðÅ & lr-AdamW/pg1‚0.0014723611480910233 & lr-AdamW/pg2‚0.0014723611480910233 trainer/global_step‚2099 % _timestamp‚1.7866592117080193e+09 _runtime‚ 560.238088355 _step‚169©d…ÖÑ0ñ+ _wandbruntime‚560  _runtime‚560ê¾)ɉò‘O ]LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 507170315776 OCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚7766549 UHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚70973440 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚262 D;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚5264 [HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚56904754233344 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 4552982528 YFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚50446121320448 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚262 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 5492441088 OCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚7766551 OCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚7047434 SGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚7047434 dQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚49909768388608 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚59395584 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 4552982528 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚429 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚321 YHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 506071892606 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 123731968 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚262 ZIDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 507170315776 XEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚57749363219968 TADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚57410826125950 KDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚59 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚4287 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚223 BDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 E8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚14809696 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚4834409 PCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚58387380 _runtime‚ 560.239879708 _step‚170 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 122905600 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 122905600 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚1768425 ^MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 536373575680 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 F>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 YHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 507128370688 IDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4876 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚41945088 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17104896 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 4628521206 F?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚59 UBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚57749363219968 Q>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚57749304169984 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚1768489 @7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚5264 M@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚59049984 SGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚7766551 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 4654602240 \IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚57242192904192 XEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚57242175799296 _LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚56904771272704 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚14813985 L@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚6602832 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚50329600 trainer/global_step‚2099 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚20643840 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 5492441088 $ _timestamp‚1.786659211709073e+09 TADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚57749304169984 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚41945088 JBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚665 IBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚66 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚209 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 122306526 K?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚7766549 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚4287 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4289 XEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚57410884513330 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚321ȼ\œ'ó–O KDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚59 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚4287 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚223 BDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 E8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚14809696 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚4834409 PCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚58387380 _runtime‚ 560.239879708 _step‚170 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 122905600 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 122905600 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚1768425 ^MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 536373575680 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 F>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 YHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 507128370688 IDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4876 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚41945088 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17104896 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 4628521206 F?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚59 UBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚57749363219968 Q>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚57749304169984 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚1768489 @7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚5264 M@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚59049984 SGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚7766551 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 4654602240 \IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚57242192904192 XEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚57242175799296 _LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚56904771272704 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚14813985 L@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚6602832 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚50329600 trainer/global_step‚2099 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚20643840 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 5492441088 $ _timestamp‚1.786659211709073e+09 TADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚57749304169984 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚41945088 JBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚665 IBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚66 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚209 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 122306526 K?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚7766549 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚4287 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4289 XEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚57410884513330 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚321 ]LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 507170315776 OCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚7766549 UHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚70973440 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚262 D;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚5264 [HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚56904754233344 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 4552982528 YFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚50446121320448 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚262 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 5492441088 OCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚7766551 OCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚7047434 SGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚7047434 dQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚49909768388608 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚59395584 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 4552982528 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚429 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚321 YHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 506071892606 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 123731968 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚262 ZIDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 507170315776 XEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚57749363219968 TADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚57410826125950ªÔ¥:Æ0ô+ _wandbruntime‚560  _runtime‚560›0R¢&õœM K>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59053056 TCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 507359439360 NADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚59053056 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 4552982528 I@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚4876 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 trainer/global_step‚2099 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 WDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚50469415657472 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 5368709120 I=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚7046434 [JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 507401387520 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17104896 ]JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚56930957066240 _step‚171 E5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5264 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 S@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚57775916629504 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚14816536 J>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚6605866 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 D=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚59 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚1769331 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 5492441088 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚262 MADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚7770102 [HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚50469482433536 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚59395584 WFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 507359439360 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚321 G@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚59 D6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚321 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 4517085184 ZGDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚57268515241984 XGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 507401387520 ]JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚57268515241984 QEDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚7770104 NBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚6605945 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 123731968 B9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚5264 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚4836614 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚1769254 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521206 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 122306526 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 123731968 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 122905600 R?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚57775857576448 bODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚49932815761408 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚20643840 `ODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 536666672128 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 5492441088 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚262 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚14820831 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚14820831 NGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚77 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 843096576 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 B6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚321 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 4654602240 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 4552982528 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚41948160 VCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚57775916629504 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚4293 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚429 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 MADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚7770104 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚209 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚41948160 SFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚66776064 % _timestamp‚1.7866592118888886e+09 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 [JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 506343813286 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 123731968 ODeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 YFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚57268498137088 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 5492441088 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 831258624 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 WFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 506302465218 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654602240 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17104896 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 NADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚58387428 I=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚7770102J4vÈaö¡M YFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚56930940026880 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚4836612 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚4293 H@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚665 R?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚57437242492098 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚262 _runtime‚ 560.419417948 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚223 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 ^KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚49932795117568 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚321 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 4517085184 ZGDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚57268515241984 XGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 507401387520 ]JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚57268515241984 QEDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚7770104 NBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚6605945 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 123731968 B9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚5264 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚4836614 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚1769254 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521206 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 122306526 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 123731968 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 122905600 R?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚57775857576448 bODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚49932815761408 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚20643840 `ODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 536666672128 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 5492441088 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚262 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚14820831 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚14820831 NGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚77 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 843096576 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 B6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚321 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 4654602240 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 4552982528 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚41948160 VCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚57775916629504 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚4293 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚429 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 MADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚7770104 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚209 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚41948160 SFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚66776064 % _timestamp‚1.7866592118888886e+09 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 [JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 506343813286 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 123731968 ODeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 YFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚57268498137088 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 5492441088 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 831258624 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 WFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 506302465218 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654602240 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17104896 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 NADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚58387428 I=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚7770102 K>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59053056 TCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 507359439360 NADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚59053056 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 4552982528 I@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚4876 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 trainer/global_step‚2099 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 WDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚50469415657472 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 5368709120 I=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚7046434 [JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 507401387520 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17104896 ]JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚56930957066240 _step‚171 E5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5264 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 S@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚57775916629504 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚14816536 J>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚6605866 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 D=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚59 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚1769331 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 5492441088 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚262 MADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚7770102 [HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚50469482433536 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚59395584 WFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 507359439360 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚321 G@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚59 DDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4876 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 [HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚58558292688896 ]LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 518757081600 L@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚6753695 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 D;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚5264 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚4287 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 831258624 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚262 IDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 122905600 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 4628521206 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 5368709120 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 5368709120 M@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚59049984 Q>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚59077007825408 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 4654602240 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚4944657 PIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚64 TADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚58731677613358 YHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 517633703214 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚321 @7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚5264 \IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚58558309793792 JBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚665 ^MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 548672389120 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚321 XEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚59077066875392 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚15156737 PDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚6753761 XEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚58731736000738 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 5368709120 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 F?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚59 _runtime‚ 573.401932216 PCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚59049984 OCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚7212536 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 _LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚58214060949504 C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚4289 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 4517085184 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚20643840 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚321 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 ]LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 517675051234 _step‚174 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 4552982528 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 PCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚58387380 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚41348020 KDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚59 % _timestamp‚1.7866592248681698e+09 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17104896 IBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚66 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚1809102 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 122306526 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚262 TADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚59077007825408 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚41945088 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚1809038 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17104896 _LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚58558309793792 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚4944659 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚209 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚59395584 YHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 518715136512 `MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚51062116384768 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚429 XEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚58558292688896 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4289 trainer/global_step‚2149›²æÐ'—O KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 _LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚58214060949504 C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚4289 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 4517085184 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚20643840 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚321 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 ]LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 517675051234 _step‚174 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 4552982528 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 PCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚58387380 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚41348020 KDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚59 % _timestamp‚1.7866592248681698e+09 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17104896 IBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚66 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚1809102 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 122306526 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚262 TADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚59077007825408 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚41945088 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚1809038 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17104896 _LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚58558309793792 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚4944659 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚209 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚59395584 YHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 518715136512 `MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚51062116384768 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚429 XEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚58558292688896 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4289 trainer/global_step‚2149 G>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4876 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 [HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚58558292688896 ]LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 518757081600 L@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚6753695 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 D;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚5264 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚4287 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 831258624 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚262 IDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 122905600 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 4628521206 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 5368709120 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 5368709120 M@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚59049984 Q>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚59077007825408 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 4654602240 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚4944657 PIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚64 TADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚58731677613358 YHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 517633703214 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚321 @7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚5264 \IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚58558309793792 JBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚665 ^MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 548672389120 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚321 XEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚59077066875392 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚15156737 PDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚6753761 XEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚58731736000738 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 5368709120 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 F?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚59 _runtime‚ 573.401932216 PCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚59049984 OCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚7212536®'O´0‚+ _wandbruntime‚573  _runtime‚573§sñt惜M C6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚15159288 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17104896 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 R?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚58758093979506 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 5492441088 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚321 [HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚51634143562752 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚223 VCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚59103620284928 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 122905600 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 4517085184 G@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚69 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 BDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59053056 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 4552982528 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚4293 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 122306526 _step‚175 ^KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚51085163757568 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 843096576 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 D=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚59 DDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 ZGDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚58584632131584 XGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 518988153344 MADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚7947752 >5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5264 EDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚6756728 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚429 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 5368709120 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚321 ]JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚58584632131584 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚20643840 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 trainer/global_step‚2149 I=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚7211536 YFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚58584615026688 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 123731968 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 ]JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚58240246743040 WFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 517864275826 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚321 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚15163583 I=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚7947752 NADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚59053056 I@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚4876 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚209 SFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚64678912 WDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚51634078883840 MADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚7215829 O8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 5368709120 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 123731968 R?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚59103561231872 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚41948160 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚4946862 NGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚67 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 831258624 VCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.OÇ,æ¼allocated‚58758152366934 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚41948160 WFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 518946205184 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521206 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚41348068 G@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚59 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚1809866 `ODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 548959161344 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 YFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚58240229703680 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 5492441088 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚4946864 S@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚59103620284928 VCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚58584615026688 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654602240 MADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚7211536 MADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚7947754 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 122905600 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚1809933 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17104896 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚15159288 B9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚5264 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚321 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 123731968 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚262 % _timestamp‚1.7866592250349853e+09 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚15163583 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0Ï)ʧ&„¡M E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 TCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 518946205184 [JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 518988153344 QEDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚7947754 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 5368709120 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 123731968 R?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚59103561231872 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚41948160 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚4946862 NGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚67 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 831258624 VCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚58758152366934 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚41948160 WFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 518946205184 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521206 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚41348068 G@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚59 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚1809866 `ODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 548959161344 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 YFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚58240229703680 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 5492441088 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚4946864 S@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚59103620284928 VCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚58584615026688 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654602240 MADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚7211536 MADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚7947754 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 122905600 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚1809933 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17104896 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚15159288 B9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚5264 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚321 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 123731968 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚262 % _timestamp‚1.7866592250349853e+09 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚15163583 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚15159288 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17104896 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 R?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚58758093979506 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 5492441088 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚321 [HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚51634143562752 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚223 VCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚59103620284928 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 122905600 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 4517085184 G@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚69 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 BDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59053056 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 4552982528 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚4293 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 122306526 _step‚175 ^KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚51085163757568 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 843096576 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 D=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚59 DDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 ZGDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚58584632131584 XGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 518988153344 MADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚7947752 >5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5264 EDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚6756728 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚429 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 5368709120 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚321 ]JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚58584632131584 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚20643840 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 trainer/global_step‚2149 I=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚7211536 YFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚58584615026688 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 123731968 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 ]JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚58240246743040 WFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 517864275826 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚321 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚15163583 I=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚7947752 NADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚59053056 I@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚4876 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚209 SFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚64678912 WDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚51634078883840 MADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚7215829 ODeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚60404711480832 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 ]LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 529236861842 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 5368709120 TADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚60052529100766 L@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚6904499 PCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚58387380 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 4517085184 G>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4876 bQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 561010842112 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚321 BDeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4289 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚41945088 IDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 4552982528 PCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚59049984 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17104896 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚15499489 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚1849645 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚321 VEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 530301902336 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚4287 PIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚53 KDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚59 D;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚5264 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚209 ]JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚52775516510720 ^MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 560966803968 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 5368709120 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚321 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚262 E8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚15495200 OCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚7377638 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 4654602240 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 4552982528 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 843096576 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚321 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 4628521206 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 5492441088 SGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚8121851 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 [HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚59523333586944 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚41348020 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 UBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚60404770530816 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 122905600 ]LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 530343847424 TADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚60404711480832 [HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚59874409578496 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚5054907 `MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚52214485024768 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 _step‚178 @7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚5264 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 _LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚59874426683392 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0Ø´Õƒ–O KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 Q>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚60404711480832 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 ]LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 529236861842 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 5368709120 TADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚60052529100766 L@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚6904499 PCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚58387380 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 4517085184 G>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4876 bQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 561010842112 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚321 BDeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4289 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚41945088 IDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 4552982528 PCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚59049984 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17104896 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚15499489 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚1849645 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚321 VEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 530301902336 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚4287 PIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚53 KDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚59 D;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚5264 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚209 ]JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚52775516510720 ^MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 560966803968 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 5368709120 C;DeviceStatsMonitor.on_train_barÊ$± tch_start/segment.all.current‚321 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚262 E8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚15495200 OCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚7377638 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 4654602240 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 4552982528 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 843096576 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚321 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 4628521206 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 5492441088 SGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚8121851 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 [HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚59523333586944 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚41348020 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 UBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚60404770530816 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 122905600 ]LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 530343847424 TADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚60404711480832 [HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚59874409578496 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚5054907 `MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚52214485024768 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 _step‚178 @7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚5264 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 _LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚59874426683392²0ü È0+ _wandbruntime‚585  _runtime‚585û Ž7¢&‘œM XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 5368709120 % _timestamp‚1.7866592373312492e+09 WFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 530532971008 B9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚5264 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 `ODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 561256754688 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 5492441088 E6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚321 MADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚7376638 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 ODeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654602240 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 5492441088 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 5368709120 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚321 TCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 530532971008 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 831258624 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 4517085184 R?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚60431264887296 MADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚8125402 J>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚6907506 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚1850453 _runtime‚ 585.861822271 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17104896 QEDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚8125404 I@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚4876 DDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 H@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚665 [HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚52798809796096 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 843096576 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 5492441088 trainer/global_step‚2199 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 122905600 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 122905600 SFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚64678912 VCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚60079003854342 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚4295 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 4654602240 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚1850394 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 123731968 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 123731968 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚44035072 \KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 561212719616 B6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚321 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚262 K>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59053056 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521206 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 5368709120 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚41948160 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚223 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 _step‚179 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17104896 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚15506335 bODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚52237553041408 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 NADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚58387428 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚262 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 122306526 D=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚59 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚4293 R?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚60078945466914 IBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚59 >5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5264 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚429 WDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚52798745117184 S@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚60431323940352 NADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚59053056 ]JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚59900749021184 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚15502040 QEDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚7380931 [JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 529467434502 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚41348068 G@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚59 VCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚59900731916288 VCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚60431323940352 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚4293 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 YFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚59549519380480 …ê§&’¡M KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 H@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚665 [HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚52798809796096 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 843096576 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 5492441088 trainer/global_step‚2199 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 122905600 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 122905600 SFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚64678912 VCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚60079003854342 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚4295 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 4654602240 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚1850394 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 123731968 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 123731968 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚44035072 \KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 561212719616 B6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚321 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚262 K>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59053056 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521206 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 5368709120 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚41948160 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚223 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 _step‚179 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17104896 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚15506335 bODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚52237553041408 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 NADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚58387428 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚262 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 122306526 D=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚59 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚4293 R?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚60078945466914 IBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚59 >5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5264 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚429 WDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚52798745117184 S@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚60431323940352 NADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚59053056 ]JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚59900749021184 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚15502040 QEDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚7380931 [JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 529467434502 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚41348068 G@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚59 VCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚59900731916288 VCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚60431323940352 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚4293 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 YFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚59549519380480 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 5368709120 % _timestamp‚1.7866592373312492e+09 WFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 530532971008 B9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚5264 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 `ODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 561256754688 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 5492441088 E6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚321 MADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚7376638 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 ODeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654602240 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 5492441088 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 5368709120 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚321 TCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 530532971008 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 831258624 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 4517085184 R?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚60431264887296 MADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚8125402 J>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚6907506 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚1850453 _runtime‚ 585.861822271 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17104896 QEDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚8125404 I@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚4876 DDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0³Ÿá«s0“+ _wandbruntime‚585  _runtime‚5857qRôo”é 0loss/FRt1/assignment_loss‚1.0531904744660905 /loss/FRt2/assignment_loss‚0.940087479971627 /loss/FRt1/detection_loss‚0.5597250210532111 /loss/FRt2/detection_loss‚0.5415165913229011 &loss/total_loss‚3.0945195668138297 epoch‚5 _runtime‚ 585.86227684 _step‚180 trainer/global_step‚2199 % _timestamp‚1.7866592373324435e+09\t•î &loss/total_loss‚3.0945195668138297 epoch‚5 _runtime‚ 585.86227684 _step‚180 trainer/global_step‚2199 % _timestamp‚1.7866592373324435e+09 0loss/FRt1/assignment_loss‚1.0531904744660905 /loss/FRt2/assignment_loss‚0.940087479971627 /loss/FRt1/detection_loss‚0.5597250210532111 /loss/FRt2/detection_loss‚0.5415165913229011´ª'¡Q0–+ _wandbruntime‚598  _runtime‚598Ê#3/Æ—À _runtime‚ 598.927680518 _step‚181 & lr-AdamW/pg1‚0.0014673027254975283 & lr-AdamW/pg2‚0.0014673027254975283 trainer/global_step‚2249 % _timestamp‚1.7866592503975968e+09©_Ù:˘Š& lr-AdamW/pg1‚0.0014673027254975283 & lr-AdamW/pg2‚0.0014673027254975283 trainer/global_step‚2249 % _timestamp‚1.7866592503975968e+09 _runtime‚ 598.927680518 _step‚181µzF•i0™+ _wandbruntime‚598  _runtime‚598Ù&fo$š’O PCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚59049984 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚429 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚44038144 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚262 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚41945088 UHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚64681984 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 843096576 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 5368709120 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚1890305 BDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚41348020 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚4287 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17104896 VEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 541888668160 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚15842241 OCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚7538453 KDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚59 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 XEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚61190526468096 IBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚71 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 4552982528 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 4517085184 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚262 @7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚5264 \IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚61190543572992 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚223 dQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚53366874308608 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 PCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚58387380 [HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚60832623263744 trainer/global_step‚2249 E8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚15837952 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 122905600 ]JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚53940163649024 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚321 K?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚8299499 [HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚61190526468096 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4289 IDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚61732415136256 OCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚8299499 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚321 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 TADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚61373380588174 F>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 4552982528 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 UBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚61732474186240 KBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚4876 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚59395584 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17104896 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 D;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚5264 ^MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 573245302272 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 5492441088 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚20643840 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 5368709120 % _timestamp‚1.7866592503987699e+09 _runtime‚ 598.932933898 ]LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 541930613248 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚5165159 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 831258624 _step‚182 TADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚61732415136256 SGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚7542740 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚209 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 IDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4876`áè'›—O JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚4287 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17104896 VEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 541888668160 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚15842241 OCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚7538453 KDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚59 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 XEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚61190526468096 IBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚71 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 4552982528 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 4517085184 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚262 @7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚5264 \IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚61190543572992 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚223 dQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚53366874308608 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 PCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚58387380 [HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚60832623263744 trainer/global_step‚2249 E8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚15837952 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 122905600 ]JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚53940163649024 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚321 K?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚8299499 [HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚61190526468096 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4289 IDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚61732415136256 OCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚8299499 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚321 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 TADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚61373380588174 F>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 4552982528 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 UBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚61732474186240 KBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚4876 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚59395584 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17104896 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 D;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚5264 ^MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 573245302272 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 5492441088 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚20643840 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 5368709120 % _timestamp‚1.7866592503987699e+09 _runtime‚ 598.932933898 ]LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 541930613248 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚5165159 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 831258624 _step‚182 TADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚61732415136256 SGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚7542740 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚209 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 IDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4876 PCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚59049984 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚429 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚44038144 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚262 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚41945088 UHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚64681984 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 843096576 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 5368709120 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚1890305 BDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚41348020¶ÖœÞ0œ+ _wandbruntime‚599  _runtime‚599½f)%¢&œM E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚262 D6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚321 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚5167362 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 S@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚61759027595776 XGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 542161684992 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚429 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 5368709120 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 >5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5264 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 4552982528 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 I=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚8303052 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 123731968 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚262 trainer/global_step‚2249 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 VCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚61216848805888 VCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚61399855341750 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 123731968 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚321 G@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚59 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚321 B9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚5264 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚209 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521206 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 5368709120 SFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚66776064 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 843096576 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 [JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 541029245110 EDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59053056 H@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚665 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚20643840 ^KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚53389901037568 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 4654602240 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17104896 R?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚61399796954322 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 4552982528 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654602240 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 5492441088 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 ZGDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚61216865910784 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚223 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚1891105 [HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚53963458146816 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚46132224 NADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚58387428 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚41348068 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 122306526 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17104896 QEDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚7546033 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 123731968 J>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚7058467 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 5368709120 _step‚183 MADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚8303054 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚41948160 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚4295 R?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚61758968542720 QEDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚8303054 MADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚7541740 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 831258624 I=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚7541740 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚15849087 WDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚53963391370752 bODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚53389921681408 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚262 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚59395584ôëK§&ž¡M K>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59053056 H@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚665 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚20643840 ^KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚53389901037568 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 4654602240 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17104896 R?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚61399796954322 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 4552982528 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654602240 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 5492441088 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 ZGDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚61216865910784 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚223 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚1891105 [HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚53963458146816 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚46132224 NADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚58387428 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚41348068 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 122306526 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17104896 QEDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚7546033 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 123731968 J>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚7058467 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 5368709120 _step‚183 MADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚8303054 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚41948160 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚4295 R?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚61758968542720 QEDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚8303054 MADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚7541740 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 831258624 I=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚7541740 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚15849087 WDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚53963391370752 bODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚53389921681408 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚262 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚59395584 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚262 D6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚321 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚5167362 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 S@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚61759027595776 XGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 542161684992 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚429 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 5368709120 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 >5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5264 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 4552982528 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 I=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚8303052 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 123731968 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚262 trainer/global_step‚2249 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 VCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚61216848805888 VCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚61399855341750 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 123731968 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚321 G@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚59 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚321 B9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚5264 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚209 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521206 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 5368709120 SFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚66776064 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 843096576 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 [JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 541029245110 EDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 UBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚63060177841664 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 4654602240 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 843096576 E8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚16180704 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 4552982528 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚5275409 PCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚59049984 JBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚665 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚209 G>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4876 OCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚8477151 OCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚7703555 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 [HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚62141912940544 % _timestamp‚1.7866592630290756e+09 Q>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚63060118791680 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 831258624 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 4517085184 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 5492441088 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 5368709120 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 5368709120 [HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚62506643357696 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚59395584 _LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚62141929979904 @7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚5264 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 122905600 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚16184993 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 ]LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 552360483058 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 123731968 F>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4289 `MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚54519222304768 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚46135296 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 123731968 IDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 UBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚63060177841664 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 4654602240 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 843096576 E8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚16180704 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 4552982528 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚5275409 PCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚59049984 JBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚665 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚209 G>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4876 OCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚8477151 OCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚7703555 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 [HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚62141912940544 % _timestamp‚1.7866592630290756e+09 Q>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚63060118791680 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 831258624 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 4517085184 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 5492441088 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 5368709120 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 5368709120 [HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚62506643357696 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚59395584 _LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚62141929979904 @7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚5264 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 122905600 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚16184993 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 ]LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 552360483058 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 123731968 F>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4289 `MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚54519222304768 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚46135296 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 123731968 IDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654602240 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚223 YFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚62168098734080 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 122306526 % _timestamp‚1.7866592631959136e+09 _runtime‚ 611.726310597 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17104896 SFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚68873216 bODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚54542290321408 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 VCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚62720706829158 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 [JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 553748450816 WFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 553706502656 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚16191839 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚5277614 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 5368709120 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚321 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚4295 MADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚8480704 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚16187544 QEDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚7711135 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚429 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚321 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 123731968 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚16187544 MADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚7711135 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 831258624 `ODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 585833034752 \KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 585784805376 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚262 ODeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚7209368 I=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚8480702 K>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59053056 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 4654602240 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚41948160 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 4517085184 trainer/global_step‚2299 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 122905600 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚1931756 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚41348068 D8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 5492441088 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚4293 S@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚63086731251200 I@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚4876 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚209 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521206 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚321 >5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5264 TCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 553706502656 EDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚16191839 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚20643840 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 4552982528 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚41948160 H@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚665 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚5277612 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚1931826 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚321PT´¬¡M E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚48229376 IBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚59 R?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚63086672198144 R?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚62720648441730 EDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654602240 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚223 YFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚62168098734080 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 122306526 % _timestamp‚1.7866592631959136e+09 _runtime‚ 611.726310597 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17104896 SFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚68873216 bODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚54542290321408 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 VCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚62720706829158 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.fre" Âáóed‚0 [JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 553748450816 WFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 553706502656 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚16191839 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚5277614 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 5368709120 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚321 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚4295 MADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚8480704 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚16187544 QEDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚7711135 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚429 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚321 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 123731968 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚16187544 MADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚7711135 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 831258624 `ODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 585833034752 \KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 585784805376 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚262 ODeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚7209368 I=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚8480702 K>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59053056 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 4654602240 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚41948160 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 4517085184 trainer/global_step‚2299 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 122905600 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚1931756 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚41348068 D8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 5492441088 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚4293 S@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚63086731251200 I@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚4876 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚209 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521206 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚321 >5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5264 TCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 553706502656 EDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚16191839 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚20643840 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 4552982528 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚41948160 H@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚665 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚5277612 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚1931826 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚321»éÓ«ÿ0­+ _wandbruntime‚611  _runtime‚611’=#q®ë 0loss/FRt1/assignment_loss‚0.9306388449328772 trainer/global_step‚2299 % _timestamp‚1.7866592631969595e+09 _runtime‚ 611.726795141 0loss/FRt2/assignment_loss‚0.9004025446394218 /loss/FRt1/detection_loss‚0.5688325618068326 /loss/FRt2/detection_loss‚0.5419296223825549 &loss/total_loss‚2.9418035737616863 epoch‚5 _step‚188™åov¯ð epoch‚5 _step‚188 0loss/FRt1/assignment_loss‚0.9306388449328772 trainer/global_step‚2299 % _timestamp‚1.7866592631969595e+09 _runtime‚ 611.726795141 0loss/FRt2/assignment_loss‚0.9004025446394218 /loss/FRt1/detection_loss‚0.5688325618068326 /loss/FRt2/detection_loss‚0.5419296223825549 &loss/total_loss‚2.9418035737616863¼Ž?o –°‚(:‹ ƒùÓ¼¬óà gpu.0.gpu‚61.0 gpu.0.memory‚11, gpu.0.memoryAllocated‚7.0288848876953125) gpu.0.memoryAllocatedBytes‚ 6037766144 gpu.0.temp‚36.0 gpu.0.powerWatts‚222.771' gpu.0.enforcedPowerLimitWatts‚500.0 gpu.0.powerPercent‚44.5542 gpu.0.smClock‚1410 gpu.0.memoryClock‚1593$ gpu.0.correctedMemoryErrors‚1955# gpu.0.uncorrectedMemoryErrors‚0“`¿•±‚(:Š ƒùÓÌ›ã proc.cpu.threads‚17 disk.drbd2165.in‚0 disk.sda3.in‚ 5881.49609375 disk.sda.in‚ 5881.49609375 disk.sda1.out‚0 disk.sda2.out‚0" proc.memory.rssMB‚ 2368.9921875 cpu‚26.568204894685373 network.recv‚ 1.278505e+06* disk./.usagePercent‚29.667499667395013' disk.nvme0c0n1.out‚3523.11376953125 disk.sda1.in‚0% disk.nvme0n1.out‚3523.11376953125 disk.dm-1.out‚ 11.79296875$ memory_percent‚65.39113521575928) proc.memory.availableMB‚ 5670.31640625 network.sent‚ 3.055028e+06 disk.drbd2165.out‚0" disk.nvme0c0n1.in‚ 167.66015625 disk.sda2.in‚0) proc.memory.percent‚14.45918083190918$ disk./.usageGB‚442.0671844482422 disk.dm-0.in‚ 167.66015625" disk.dm-0.out‚3523.11376953125 disk.sda3.out‚ 11.79296875 disk.sda.out‚ 11.79296875 disk.nvme0n1.in‚ 167.66015625 disk.dm-1.in‚ 5881.49609375|Þ,0²+ _wandbruntime‚624  _runtime‚624Ì»À6ij¾ _step‚189 % lr-AdamW/pg1‚0.001463699908193046 % lr-AdamW/pg2‚0.001463699908193046 trainer/global_step‚2349 % _timestamp‚1.7866592758429692e+09 _runtime‚ 624.374932424(ZÉ´Ã % lr-AdamW/pg2‚0.001463699908193046 trainer/global_step‚2349 % _timestamp‚1.7866592758429692e+09 _runtime‚ 624.374932424 _step‚189 % lr-AdamW/pg1‚0.001463699908193046½jJÀé0µ+ _wandbruntime‚624  _runtime‚624üOV˜'¶’O K?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚8654799 YFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚56269421070336 _LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚63451219656704 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 123731968 M@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚59049984 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17104896 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 IDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4876 JBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚665 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 KDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚59 trainer/global_step‚2349 IDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚64387822447104 ]LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 565104144896 SGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚7872944 [HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚63451202617344 YHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 563880945646 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚5385657 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚41945088 F>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 UBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚64387881497088 OCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚7868657 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚41348020 PCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚59049984 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚4287 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 122905600 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 E8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚16523456 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚4287 \IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚63822777352192 [HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚63822760247296 YHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 565062199808 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4289 ZIDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 565104144896 BDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 ]LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 563922293666 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 5492441088 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚321 % _timestamp‚1.7866592758436532e+09 _runtime‚ 624.375042793 OCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚7872944 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 4654602240 bQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 597876260864 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 L@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚7357174 _step‚190«ê¿<'·—O MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 4552982528 OCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚8654799 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 ]LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 563922293666 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 5492441088 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚321 % _timestamp‚1.7866592758436532e+09 _runtime‚ 624.375042793 OCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚7872944 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 4654602240 bQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 597876260864 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 L@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚7357174 _step‚190 K?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚8654799 YFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚56269421070336 _LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚63451219656704 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 123731968 M@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚59049984 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17104896 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 IDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4876 JBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚665 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 KDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚59 trainer/global_step‚2349 IDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚64387822447104 ]LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 565104144896 SGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚7872944 [HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚63451202617344 YHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 563880945646 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚5385657 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚41945088 F>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 UBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚64387881497088 OCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚7868657 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚41348020 PCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚59049984 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚4287 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 122905600 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 E8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚16523456 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚4287 \IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚63822777352192 [HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚63822760247296 YHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 565062199808 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4289 ZIDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 565104144896 B6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚321 WFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 564111518258 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 VCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚64414434906624 ^KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚55694638317568 R?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚64041499929138 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 5368709120 _runtime‚ 624.542688701 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 5492441088 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚4293 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 4552982528 `ODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 598120076288 MADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚7876237 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17104896 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 SFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚64678912 E?DeviceStatsMonitor.on_train_bat`„T‹ ch_end/oversize_allocations.peak‚0 E8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚321 ]JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚63849099689984 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚1972306 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚44035072 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚321 ZGDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚63849099689984 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 5492441088 >5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5264 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 IBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚59 EDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚16534591 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚5387864 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚41348068 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 D=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚59 S@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚64414434906624 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚16534591 DDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521206 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚4295 \KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 598076041216 NADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚58387428 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 123731968 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 123731968 WFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 565293268480 MADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚8658352 VCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚64041558316566 YFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚63477388410880 NADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚59053056 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654602240 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 4517085184 R?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚64414375853568 H@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚665 bODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚55694658961408 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 831258624 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚262 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 122905600 ODeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 G@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚59 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 122306526 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 trainer/global_step‚2349 I=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚7871944 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚16530296 BDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59053056 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚41948160 MADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚7871944 NGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚68 QEDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚7876237 J>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚73601688éãЧ&º¡M SFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚64678912 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 E8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚321 ]JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚63849099689984 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚1972306 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚44035072 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚321 ZGDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚63849099689984 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 5492441088 >5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5264 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 IBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚59 EDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚16534591 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚5387864 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚41348068 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 D=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚59 S@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚64414434906624 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚16534591 DDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521206 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚4295 \KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 598076041216 NADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚58387428 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 123731968 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 123731968 WFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 565293268480 MADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚8658352 VCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚64041558316566 YFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚63477388410880 NADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚59053056 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654602240 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 4517085184 R?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚64414375853568 H@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚665 bODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚55694658961408 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 831258624 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚262 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 122905600 ODeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 G@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚59 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 122306526 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 trainer/global_step‚2349 I=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚7871944 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚16530296 BDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59053056 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚41948160 MADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚7871944 NGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚68 QEDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚7876237 J>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚7360168 MADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚8658354 YFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚63849082585088 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚321 WFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 564111518258 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 VCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚64414434906624 ^KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚55694638317568 R?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚64041499929138 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 5368709120 _runtime‚ 624.542688701 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 5492441088 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚4293 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 4552982528 `ODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 598120076288 MADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚7876237 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17104896 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2¿ß ¸0»+ _wandbruntime‚624  _runtime‚624JýQ|o¼é epoch‚5 _step‚192 /loss/FRt1/assignment_loss‚0.885128869796485 0loss/FRt2/assignment_loss‚0.9305479113860765 &loss/total_loss‚2.8502127856935537 trainer/global_step‚2349 % _timestamp‚1.7866592760134027e+09 _runtime‚ 624.543396598 /loss/FRt1/detection_loss‚0.5330908906916509 .loss/FRt2/detection_loss‚0.501445113819341nÙ-çt½î .loss/FRt2/detection_loss‚0.501445113819341 epoch‚5 _step‚192 /loss/FRt1/assignment_loss‚0.885128869796485 0loss/FRt2/assignment_loss‚0.9305479113860765 &loss/total_loss‚2.8502127856935537 trainer/global_step‚2349 % _timestamp‚1.7866592760134027e+09 _runtime‚ 624.543396598 /loss/FRt1/detection_loss‚0.5330908906916509À’œ–¾‚(:‹ уùÓž¦¤á gpu.0.gpu‚60.0 gpu.0.memory‚11, gpu.0.memoryAllocated‚7.0288848876953125) gpu.0.memoryAllocatedBytes‚ 6037766144 gpu.0.temp‚36.0 gpu.0.powerWatts‚212.811' gpu.0.enforcedPowerLimitWatts‚500.0 gpu.0.powerPercent‚42.5622 gpu.0.smClock‚1410 gpu.0.memoryClock‚1593$ gpu.0.correctedMemoryErrors‚1955# gpu.0.uncorrectedMemoryErrors‚0-æ)”¿‚(:‰ уùÓÄÎä) proc.memory.percent‚14.45918083190918 proc.cpu.threads‚17 network.sent‚ 3.129948e+06 network.recv‚ 1.311279e+06 disk.sda.out‚ 13.69921875 disk.sda2.out‚0 disk.dm-1.in‚ 5881.49609375 disk.dm-1.out‚ 13.69921875" disk.dm-0.out‚3693.59326171875) proc.memory.availableMB‚ 5670.62890625) disk./.usagePercent‚29.83876153959081 disk.sda3.in‚ 5881.49609375 disk.sda3.out‚ 13.69921875 disk.sda.in‚ 5881.49609375 disk.nvme0n1.in‚ 167.66015625 disk.dm-0.in‚ 167.66015625" proc.memory.rssMB‚ 2368.9921875 cpu‚26.581221077870875" disk.nvme0c0n1.in‚ 167.66015625' disk.nvme0c0n1.out‚3693.59326171875 disk.sda1.in‚0 disk.drbd2165.out‚0$ disk./.usageGB‚444.6191101074219 disk.sda1.out‚0 disk.sda2.in‚0% disk.nvme0n1.out‚3693.59326171875 disk.drbd2165.in‚0$ memory_percent‚65.38922786712646¥è÷_0À+ _wandbruntime‚637  _runtime‚637AU`ÝÆÁÀ trainer/global_step‚2399 % _timestamp‚1.7866592885532293e+09 _runtime‚ 637.083343174 _step‚193 & lr-AdamW/pg1‚0.0014618296815351504 & lr-AdamW/pg2‚0.0014618296815351504ƒËÂÅ _runtime‚ 637.083343174 _step‚193 & lr-AdamW/pg1‚0.0014618296815351504 & lr-AdamW/pg2‚0.0014618296815351504 trainer/global_step‚2399 % _timestamp‚1.7866592885532293e+09Á õ|ä0Ã+ _wandbruntime‚637  _runtime‚637Rˆ%—'Ä‘O _step‚194 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 4552982528 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚59395584 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 E8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚16866208 @7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚5264 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 XEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚65335993437778 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 5492441088 ZIDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 576690910720 [HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚65138877136896 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 4552982528 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚321 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 IDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 KDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚59 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 4654602240 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚41945088 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚4287 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 ^MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 610143674880 PDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚7508381 OCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚8038046 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚4287 SGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚8038046 F?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚59 trainer/global_step‚2399 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚209 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚20643840 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 4628521206 F>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚41348020 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 122306526 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚46135296 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 123731968 YHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 576648965632 UBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚65715585152512 JBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚665 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚2012472 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 IBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚76 G>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4876 dQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚56823980228608 bQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 610189810176 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 D;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚5264 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 831258624 PCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚58387380 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚262 `MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚56823959584768 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚321 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 % _timestamp‚1.7866592885544307e+09 C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚4289 OCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚8832451 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 4654602240 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17104896 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 4517085184 _runtime‚ 637.08486528 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 YHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 575442756254 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 5368709120 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚321 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚262 L@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚7508305 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 M@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚59049984 K?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚8033759 KBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚4876 ]JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚57434170038784 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 123731968 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚16870497 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚223 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 K?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚8832449 XEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚65138877136896 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 OCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚8033759 PCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚59049984 TADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚65715526102528 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚5495909 UHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚66779136 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 [HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚64760492294144 \IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚65138894241792 XEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚65715585152512 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 122905600 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4289 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 _LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚64760509333504 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 5368709120 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 5368709120 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17104896 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 ]LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 575484104274 IDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚65715526102528 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 843096576 BDeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4289 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 _LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚64760509333504 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 5368709120 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 5368709120 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17104896 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 ]LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 575484104274 IDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚65715526102528 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 843096576 BDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 KDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚59 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 4654602240 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚41945088 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚4287 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 ^MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 610143674880 PDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚7508381 OCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚8038046 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚4287 SGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚8038046 F?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚59 trainer/global_step‚2399 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚209 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚20643840 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 4628521206 F>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚41348020 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 122306526 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚46135296 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 123731968 YHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 576648965632 UBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚65715585152512 JBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚665 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚2012472 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 IBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚76 G>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4876 dQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚56823980228608 bQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 610189810176 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 D;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚5264 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 831258624 PCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚58387380 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚262 `MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚56823959584768 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚321 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 % _timestamp‚1.7866592885544307e+09 C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚4289 OCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚8832451 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 4654602240 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17104896 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 4517085184 _runtime‚ 637.08486528 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 YHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 575442756254 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 5368709120 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚321 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚262 L@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚7508305 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0ºÕó0Æ+ _wandbruntime‚637  _runtime‚637Â…2¢&ÇœM C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 5368709120 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚4293 K>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59053056 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚41948160 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 122905600 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 5492441088 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 123731968 XGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 576921982464 TCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 576880034304 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚41948160 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 123731968 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚4293 MADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚8037046 bODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚56847027601408 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 R?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚65362351416546 YFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚64786678087680 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚321 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚262 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚2013270 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 831258624 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 I=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚8037046 ZGDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚65165216579584 R?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚65742079508992 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚16877343 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚20643840 G@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚59 D=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚59 DDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 VCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚65742138562048 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 % _timestamp‚1.7866592887476778e+09 _step‚195 O5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5264 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 MADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚8041339 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚59395584 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚321 EDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654602240 ^KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚56847006957568 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 5368709120 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚321 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚41348068 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 5492441088 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 4654602240 QEDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚8836004 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚5498112 SFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚64678912 WDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚57457398088192 BDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚7511316 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚262 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚262 _runtime‚ 637.278182231 [JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 576921982464 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 122905600 NBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚7511384 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17104896 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 `ODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 610435165696 [JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 575714676934 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 H@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚665 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚209 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521206 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 4517085184 S@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚65742138562048 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17104896 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 VCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚65165199474688 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 QEDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚8041339 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚16877343 NADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚59053056 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚429 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚5498114 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 843096576 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 123731968 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚16873048 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚16873048 MADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚8836004 trainer/global_step‚2399 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 45529825284Œ­§&È¡M BDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚7511316 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚262 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚262 _runtime‚ 637.278182231 [JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 576921982464 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 122905600 NBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚7511384 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17104896 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 `ODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 610435165696 [JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 575714676934 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 H@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚665 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚209 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521206 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 4517085184 S@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚65742138562048 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17104896 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 VCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚65165199474688 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 QEDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚8041339 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚16877343 NADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚59053056 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚429 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚5498114 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 843096576 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 123731968 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚16873048 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚16873048 MADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚8836004 trainer/global_step‚2399 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 4552982528 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 5368709120 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚4293 K>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59053056 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚41948160 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 122905600 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 5492441088 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 123731968 XGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 576921982464 TCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 576880034304 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚41948160 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 123731968 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚4293 MADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚8037046 bODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚56847027601408 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 R?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚65362351416546 YFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚64786678087680 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚321 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚262 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚2013270 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 831258624 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 I=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚8037046 ZGDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚65165216579584 R?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚65742079508992 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚16877343 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚20643840 G@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚59 D=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚59 DDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 VCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚65742138562048 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 % _timestamp‚1.7866592887476778e+09 _step‚195 O5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5264 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 MADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚8041339 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚59395584 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚321 EDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654602240 ^KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚56847006957568 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 5368709120 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚321 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚41348068 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 5492441088 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 4654602240 QEDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚8836004 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚5498112 SFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚64678912 WDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚57457398088192Ãj´¤Ë0É+ _wandbruntime‚637  _runtime‚637]^ÝGoÊé %loss/total_loss‚3.190680797910345 trainer/global_step‚2399 $ _timestamp‚1.786659288748836e+09 _runtime‚ 637.278725738 _step‚196 0loss/FRt1/assignment_loss‚1.1280573624108032 0loss/FRt2/assignment_loss‚0.9646772366980267 /loss/FRt1/detection_loss‚0.6004547391309615 epoch‚5 /loss/FRt2/detection_loss‚0.4974914596705535*“tËî trainer/global_step‚2399 $ _timestamp‚1.786659288748836e+09 _runtime‚ 637.278725738 _step‚196 0loss/FRt1/assignment_loss‚1.1280573624108032 0loss/FRt2/assignment_loss‚0.9646772366980267 /loss/FRt1/detection_loss‚0.6004547391309615 epoch‚5 /loss/FRt2/detection_loss‚0.4974914596705535 %loss/total_loss‚3.190680797910345Ļݭ<0Ì+ _wandbruntime‚642  _runtime‚642¿£Ö¿ÛÍÕ *loss/val_total_loss‚2.8941315136709567 particle/accuracy_1_of_1‚1 6validation_average_jet_accuracy‚0.2674332559108734 *jet/accuracy_1_of_1‚0.2630031406879425 +jet/accuracy_2_of_2‚0.16780833899974823 _step‚197 /Purity/2FRt/event_purity‚0.3229728639125824 /particle/accuracy_2_of_2‚0.6887921690940857 +validation_accuracy‚0.16780833899974823 _runtime‚ 642.41348105 2Purity/*FRt/event_proportion‚0.669805645942688 -Purity/2FRt/FRt_purity‚0.4712716042995453 /particle/accuracy_1_of_2‚0.9289578199386597 +particle/sensitivity‚0.6616572141647339 'particle/f_score‚0.5443063378334045 .Purity/*FRt/FRt_purity‚0.40276607871055603 /Purity/1FRt/event_purity‚0.3938877582550049 -Purity/1FRt/FRt_purity‚0.3938877582550049 (particle/accuracy‚0.6068289875984192 +particle/specificity‚0.5766667127609253 trainer/global_step‚2399 0Purity/*FRt/event_purity‚0.38973870873451233 Purity/0FRt/event_purity‚1 2Purity/2FRt/event_proportion‚0.040062315762043 epoch‚5 3Purity/0FRt/event_proportion‚0.3301943838596344 3Purity/1FRt/event_proportion‚0.6297432780265808 *jet/accuracy_1_of_2‚0.5055831074714661 % _timestamp‚1.7866592938834164e+094ŒjàÎÚ _step‚197 /Purity/2FRt/event_purity‚0.3229728639125824 /particle/accuracy_2_of_2‚0.6887921690940857 +validation_accuracy‚0.16780833899974823 _runtime‚ 642.41348105 2Purity/*FRt/event_proportion‚0.669805645942688 -Purity/2FRt/FRt_purity‚0.4712716042995453 /particle/accuracy_1_of_2‚0.9289578199386597 +particle/sensitivity‚0.6616572141647339 'particle/f_score‚0.5443063378334045 .Purity/*FRt/FRt_purity‚0.40276607871055603 /Purity/1FRt/event_purity‚0.3938877582550049 -Purity/1FRt/FRt_purity‚0.3938877582550049 (particle/accuracy‚0.6068289875984192 +particle/specificity‚0.5766667127609253 trainer/global_step‚2399 0Purity/*FRt/event_purity‚0.38973870873451233 Purity/0FRt/event_purity‚1 2Purity/2FRt/event_proportion‚0.040062315762043 epoch‚5 3Purity/0FRt/event_proportion‚0.3301943838596344 3Purity/1FRt/event_proportion‚0.6297432780265808 *jet/accuracy_1_of_2‚0.5055831074714661 % _timestamp‚1.7866592938834164e+09 *loss/val_total_loss‚2.8941315136709567 particle/accuracy_1_of_1‚1 6validation_average_jet_accuracy‚0.2674332559108734 *jet/accuracy_1_of_1‚0.2630031406879425 +jet/accuracy_2_of_2‚0.16780833899974823ŵѿ–Ï‚(:‹ àƒùÓê °á gpu.0.gpu‚64.0 gpu.0.memory‚12, gpu.0.memoryAllocated‚7.0288848876953125) gpu.0.memoryAllocatedBytes‚ 6037766144 gpu.0.temp‚35.0 gpu.0.powerWatts‚170.686' gpu.0.enforcedPowerLimitWatts‚500.0 gpu.0.powerPercent‚34.1372 gpu.0.smClock‚1410 gpu.0.memoryClock‚1593$ gpu.0.correctedMemoryErrors‚1955# gpu.0.uncorrectedMemoryErrors‚0yìD'±Ð‚(:‚ àƒùÓüØã disk.sda3.out‚ 14.05859375& disk.nvme0c0n1.out‚5234.9365234375 disk.sda1.out‚0# proc.memory.rssMB‚ 2377.19140625 proc.cpu.threads‚17$ disk./.usageGB‚444.3112030029297! disk.dm-0.out‚5234.9365234375 disk.sda.out‚ 14.05859375$ disk.nvme0n1.out‚5234.9365234375$ memory_percent‚65.44020175933838 cpu‚26.62527669401533 network.recv‚ 1.341065e+06 disk.dm-1.in‚ 5881.49609375 disk.nvme0c0n1.in‚ 184.171875 disk.sda1.in‚0) proc.memory.availableMB‚ 5662.27734375* proc.memory.percent‚14.509224891662598 network.sent‚ 3.201841e+06* disk./.usagePercent‚29.818097635457068 disk.dm-1.out‚ 14.05äwÓÜ859375 disk.drbd2165.out‚0 disk.sda.in‚ 5881.49609375 disk.sda2.in‚0 disk.sda2.out‚0 disk.nvme0n1.in‚ 184.171875 disk.drbd2165.in‚0 disk.dm-0.in‚ 184.171875 disk.sda3.in‚ 5881.49609375HÀ7e0Ñ+ _wandbruntime‚656  _runtime‚656®„ÓÄÒ¾ % lr-AdamW/pg1‚0.001459913735386972 % lr-AdamW/pg2‚0.001459913735386972 trainer/global_step‚2449 % _timestamp‚1.7866593080268488e+09 _runtime‚ 656.556910878 _step‚198½&šÀÉÓà % _timestamp‚1.7866593080268488e+09 _runtime‚ 656.556910878 _step‚198 % lr-AdamW/pg1‚0.001459913735386972 % lr-AdamW/pg2‚0.001459913735386972 trainer/global_step‚2449ÆøàÛr0Ô+ _wandbruntime‚656  _runtime‚656 âwŒ˜'Õ’O ZIDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 592345402880 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 831258624 ^MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 626507764224 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚41348020 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 _LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚66450698502144 ]LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 591111886988 IDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 _runtime‚ 656.558193782 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 _LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚66845948346368 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 4552982528 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚41945088 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚429 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚2062243 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 E8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚17291758 TADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚67041752001752 _step‚199 OCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚9070707 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17104896 YHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 592303457792 ]JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚58902173075968 UHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚66779136 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 4628521206 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 5368709120 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 123731968 PCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚59049984 ]LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 592345402880 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚262 IBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚59 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚4287 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 122905600 IDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 Q>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚67438234699264 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚17296047 D;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚5264 SGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚9070707 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 843096576 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 @7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚5264 M@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚59049984 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 BDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4876 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚321 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 XEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚66845931241472 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4289 PIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚64 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 5492441088 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚321 KDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚59 trainer/global_step‚2449 PDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚7709016 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚321 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 5492441088 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚209 `MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚58275598532608 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚223 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 5368709120 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚262 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 122306526 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 4654602240 JBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚665 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚5646773 YFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚58902106296832 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚262 C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚4289 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17104896>ã'Ö—O @7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚5264 M@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚59049984 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 BDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4876 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚321 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 XEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚66845931241472 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4289 PIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚64 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 5492441088 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚321 KDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚59 trainer/global_step‚2449 PDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚7709016 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚321 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 5492441088 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚209 `MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚58275598532608 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚223 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 5368709120 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚262 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 122306526 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 4654602240 JBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚665 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚5646773 YFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚58902106296832 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚262 C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚4289 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17104896 ZIDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 592345402880 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 831258624 ^MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 626507764224 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚41348020 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 _LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚66450698502144 ]LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 591111886988 IDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 _runtime‚ 656.558193782 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 _LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚66845948346368 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 4552982528 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚41945088 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚429 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚2062243 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 E8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚17291758 TADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚67041752001752 _step‚199 OCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚9070707 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17104896 YHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 592303457792 ]JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚58902173075968 UHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚66779136 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 4628521206 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 5368709120 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 123731968 PCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚59049984 ]LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 592345402880 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚262 IBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚59 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚4287 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 122905600 IDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 Q>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚67438234699264 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚17296047 D;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚5264 SGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚9070707 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 843096576 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0ÇWýPÉ0×+ _wandbruntime‚656  _runtime‚656!#=ô¢&ØœM `ODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 626798599680 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 I=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚8224340 ZGDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚66872270684160 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 122905600 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚20643840 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 831258624 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17104896 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654602240 EDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚17302893 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 122306526 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 4552982528 B9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚5264 H@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚665 R?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚67464788105728 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 G@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚59 QEDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚8228633 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚2063018 [HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚58925465148928 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 123731968 I=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚9074258 QEDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚9074260 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 MADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚9074260 DDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚321 NADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚59053056 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚321 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 843096576 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 5492441088 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚41948160 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚2063087 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 4517085184 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 123731968 _step‚200 XGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 592576474624 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚48229376 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 NADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚58387428 [JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 591342459648 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚262 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚4293 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 YFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚66476867256320 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚17302893 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 4552982528 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 VCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚67068226755328 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 K>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59053056 WFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 592534526464 NGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚69 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 BDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521206 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 I@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚4876 NBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚7712065 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 5368709120 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 5368709120 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚321 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚4295 J>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚7711994 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 D=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚59 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚17298598 VCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚67464847158784 [JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 592576474624 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚262 YFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚66872253579264 MADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚9074258 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚429 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚59395584 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚5648978 ^KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚58298645905408 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 _runtime‚ 656.740428312 >5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5264 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17104896 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚41948160 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚209 ]JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚66476884295680 WFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 591301111580 EDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚321 NADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚59053056 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚321 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 843096576 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 5492441088 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚41948160 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚2063087 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 4517085184 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 123731968 _step‚200 XGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 592576474624 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚48229376 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 NADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚58387428 [JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 591342459648 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚262 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚4293 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 YFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚66476867256320 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚17302893 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 4552982528 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 VCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚67068226755328 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 K>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59053056 WFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 592534526464 NGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚69 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 BDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521206 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 I@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚4876 NBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚7712065 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 5368709120 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 5368709120 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚321 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚4295 J>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚7711994 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 D=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚59 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚17298598 VCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚67464847158784 [JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 592576474624 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚262 YFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚66872253579264 MADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚9074258 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚429 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚59395584 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚5648978 ^KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚58298645905408 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 _runtime‚ 656.740428312 >5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5264 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17104896 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚41948160 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚209 ]JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚66476884295680 WFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 591301111580 EDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654602240 EDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚17302893 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 122306526 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 4552982528 B9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚5264 H@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚665 R?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚67464788105728 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 G@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚59 QEDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚8228633 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚2063018 [HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚58925465148928 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 123731968 I=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚9074258 QEDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚9074260 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 MADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚9074260 DDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚68765938354688 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17104896 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚41945088 YHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 603890223616 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 5492441088 IBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚59 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚20643840 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 TADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚68362603489160 VEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 603890223616 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17104896 [HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚68162048131072 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚223 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 843096576 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚321 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4289 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 [HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚67759971139584 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚262 C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚4289 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚429 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚4287 TADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚68765938354688 _runtime‚ 669.444888444 _step‚203 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 4552982528 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚17638799 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚2102839 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 122306526 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 5492441088 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚262 UBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚68765997404672 dQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚59427987816448 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 % _timestamp‚1.7866593209117386e+09 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚41348020 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 IDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4876 ]LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 603932168704 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚321 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚262 OCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚8390442 ]JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚60066808172032 trainer/global_step‚2499 XEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚68765997404672 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 123731968 PCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚59049984 D;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚5264 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚46135296 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 _LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚67759988178944 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 SGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚9248357 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚4287 F>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 ZIDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 603932168704 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 5368709120 OCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚9248357 K?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚9248355 K?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚8386155 XEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚68162048131072 UHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚66779136 YFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚60066741392896 XEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚68362661876540 `MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚59427967172608 F?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚59}žŸ¥'ä—O GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚4287 TADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚68765938354688 _runtime‚ 669.444888444 _step‚203 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 4552982528 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚17638799 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚2102839 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 122306526 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 5492441088 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚262 UBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚68765997404672 dQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚59427987816448 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 % _timestamp‚1.7866593209117386e+09 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚41348020 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 IDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4876 ]LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 603932168704 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚321 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚262 OCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚8390442 ]JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚60066808172032 trainer/global_step‚2499 XEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚68765997404672 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 123731968 PCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚59049984 D;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚5264 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚46135296 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 _LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚67759988178944 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 SGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚9248357 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚4287 F>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 ZIDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 603932168704 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 5368709120 OCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚9248357 K?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚9248355 K?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚8386155 XEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚68162048131072 UHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚66779136 YFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚60066741392896 XEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚68362661876540 `MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚59427967172608 F?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚59 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 122905600 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 PIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚67 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 KDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚59 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 4654602240 OCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚8386155 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚5757023 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚209 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚321 M@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚59049984 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 OCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚9248355 BDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚68765938354688 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17104896 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚41945088 YHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 603890223616 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 5492441088 IBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚59 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚20643840 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 TADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚68362603489160 VEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 603890223616 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17104896 [HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚68162048131072 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚223 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 843096576 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚321 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4289 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 [HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚67759971139584 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚262 C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚4289 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚429Ëô…h/0å+ _wandbruntime‚669  _runtime‚669·@(¨ZæœM MADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚9251910 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 4654602240 [HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚60090099327488 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 5492441088 IBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚59 I=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚8389442 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17104896 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 [JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 602904270256 WFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 602862922188 ODeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚7862819 R?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚6838901985530VÈ2H 8 ]JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚67786173972480 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚2103655 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 5492441088 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚262 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 NBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚7862883 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 831258624 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 122306526 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚4295 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 123731968 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17104896 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 123731968 `ODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 639064138240 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 4517085184 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 XGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 604163240448 VCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚68792550814208 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 122905600 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚17645645 ^KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚59451014545408 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚41948160 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚209 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚20643840 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚59395584 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 S@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚68792550814208 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654602240 QEDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚8393735 G@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚64 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚5759226 SFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚66776064 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚41348068 I=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚9251908 MADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚8393735 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 4552982528 [JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 604163240448 MADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚8389442 I@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚4876 YFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚67786156933120 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 123731968 TCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 604121292288 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚4293 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 MADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚9251908 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 5492441088 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚321 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚262 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚17641350 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521206 D=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚59 K>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59053056 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚223 \KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 639018006016 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 5368709120 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚17641350 >5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5264 D6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚321 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚262 G@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚59 % _timestamp‚1.7866593210727146e+09 VCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚68188370468864 NGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚62 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 4552982528 WFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 604121292288 B9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚5264 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 5368709120 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 5368709120 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚17645645 NADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚59053056 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 ZGDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚68188387573760 ]JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚68188387573760 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚5759228 trainer/global_step‚2499 _runtime‚ 669.602978421 bODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚59451035189248 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚46132224 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚4293 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 843096576 _step‚204 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚321=¾d§&ç¡M I=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚8389442 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17104896 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 [JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 602904270256 WFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 602862922188 ODeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚7862819 R?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚68389019855308 ]JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚67786173972480 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚2103655 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 5492441088 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚262 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 NBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚7862883 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 831258624 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 122306526 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚4295 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 123731968 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17104896 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 123731968 `ODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 639064138240 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 4517085184 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 XGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 604163240448 VCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚68792550814208 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 122905600 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚17645645 ^KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚59451014545408 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚41948160 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚209 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚20643840 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚59395584 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 S@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚68792550814208 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654602240 QEDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚8393735 G@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚64 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚5759226 SFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚66776064 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚41348068 I=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚9251908 MADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚8393735 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 4552982528 [JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 604163240448 MADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚8389442 I@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚4876 YFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚67786156933120 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 123731968 TCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 604121292288 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚4293 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 MADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚9251908 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 5492441088 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚321 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚262 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚17641350 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521206 D=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚59 K>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59053056 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚223 \KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 639018006016 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 5368709120 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚17641350 >5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5264 D6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚321 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚262 G@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚59 % _timestamp‚1.7866593210727146e+09 VCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚68188370468864 NGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚62 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 4552982528 WFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 604121292288 B9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚5264 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 5368709120 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 5368709120 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚17645645 NADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚59053056 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 ZGDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚68188387573760 ]JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚68188387573760 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚5759228 trainer/global_step‚2499 _runtime‚ 669.602978421 bODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚59451035189248 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚46132224 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚4293 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 843096576 _step‚204 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚321 MADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚9251910 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 4654602240 [HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚60090099327488 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 5492441088 IBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚59Ìîò®Å0è+ _wandbruntime‚669  _runtime‚669†òã„péê /loss/FRt1/detection_loss‚0.5835778594159098 epoch‚6 trainer/global_step‚2499 _step‚205 .loss/FRt2/detection_loss‚0.525084213327171 &loss/total_loss‚2.9431119458488464 % _timestamp‚1.7866593210736926e+09 _runtime‚ 669.603488936 0loss/FRt1/assignment_loss‚0.9485860279561623 0loss/FRt2/assignment_loss‚0.8858638451496035XW‰¶uêï &loss/total_loss‚2.9431119458488464 % _timestamp‚1.7866593210736926e+09 _runtime‚ 669.603488936 0loss/FRt1/assignment_loss‚0.9485860279561623 0loss/FRt2/assignment_loss‚0.8858638451496035 /loss/FRt1/detection_loss‚0.5835778594159098 epoch‚6 trainer/global_step‚2499 _step‚205 .loss/FRt2/detection_loss‚0.525084213327171Í}5Ü÷–ë‚(:‹ þƒùÓ†¿¹á gpu.0.gpu‚61.0 gpu.0.memory‚11, gpu.0.memoryAllocated‚7.0288848876953125) gpu.0.memoryAllocatedBytes‚ 6037766144 gpu.0.temp‚34.0 gpu.0.powerWatts‚129.029' gpu.0.enforcedPowerLimitWatts‚500.0 gpu.0.powerPercent‚25.8058 gpu.0.smClock‚1410 gpu.0.memoryClock‚1593$ gpu.0.correctedMemoryErrors‚1955# gpu.0.uncorrectedMemoryErrors‚0w.c8Œì‚(: þƒùÓ°¿ôã' disk.nvme0c0n1.out‚5715.48583984375 disk.sda1.out‚0 disk.sda2.in‚0 disk.sda2.out‚0 network.recv‚ 1.405932e+06* disk./.usagePercent‚29.822441571298057 disk.nvme0n1.in‚ 184.171875% disk.nvme0n1.out‚5715.48583984375 disk.dm-1.in‚ 5881.49609375" disk.dm-0.out‚5715.48583984375$ memory_percent‚65.44544696807861# proc.memory.rssMB‚ 2377.19140625 disk.sda.in‚ 5881.49609375 disk.sda.out‚ 14.7734375 disk.dm-1.out‚ 14.7734375 disk.drbd2165.in‚0 disk.drbd2165.out‚0 cpu‚26.64385566461762 disk.sda3.in‚ 5881.49609375 disk.sda1.in‚0 disk.dm-0.in‚ 184.171875) proc.memory.availableMB‚ 5661.41796875* proc.memory.percent‚14.509224891662598 proc.cpu.threads‚17 network.sent‚ 3.35211e+06$ disk./.usageGB‚444.3759307861328 disk.sda3.out‚ 14.7734375 disk.nvme0c0n1.in‚ 184.171875!CR0í+ _wandbruntime‚681  _runtime‚681)^ŠBÆîÀ trainer/global_step‚2549 % _timestamp‚1.7866593331477978e+09 _runtime‚ 681.679874194 _step‚206 & lr-AdamW/pg1‚0.0014559451797789035 & lr-AdamW/pg2‚0.0014559451797789035W«u ËïÅ % _timestamp‚1.7866593331477978e+09 _runtime‚ 681.679874194 _step‚206 & lr-AdamW/pg1‚0.0014559451797789035 & lr-AdamW/pg2‚0.0014559451797789035 trainer/global_step‚2549Îÿ ™0ð+ _wandbruntime‚681  _runtime‚681Ôê©¢—'ñ‘O TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17104896 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚41940992 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 XEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚70093701060096 PIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚72 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 4628521206 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚321 F?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚59 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 4552982528 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 122905600 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 IDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 UBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚70093701060096 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚41945088 [HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚69478165020672 ]LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 615518934528 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚429 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 @7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚5264 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚5867271 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 PCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚59049984 PCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚58387380 OCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚8555544 TADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚70093642010112 `MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚60580335812608 OCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚9426007 IDeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4289 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 [HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚69069260816384 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚223 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚2143822 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 5368709120 KDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚59 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 5368709120 E8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚17977262 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 4552982528 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚4287 Q>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚70093642010112 SGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚8555544 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 5368709120 KBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚4876 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚2143750 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 843096576 bQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 651095080960 K?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚9426005 VEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 615476989440 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚59395584 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 $ _timestamp‚1.786659333148441e+09 G>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4876 _LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚69478182125568 SGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚9426007 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚5867273 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚20643840 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚321 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚321 IBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚59 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 122905600 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 IBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚74 UHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚62584832 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 ZIDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 615518934528 JBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚665 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚262 _step‚207 XEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚69478165020672 L@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚8011021 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 5492441088 K?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚8551257 D;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚5264 OCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚9426005 ]JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚61231451537408 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚262 C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚4289 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 4654602240 trainer/global_step‚2549 YHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 615476989440 ^MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 651053139968 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 5492441088 M@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚59049984 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17104896 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚4287 dQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚60580356456448 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 4517085184 YFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚61231388952576 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 123731968 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 123731968 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 123731968 OCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚8551257 TADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚69683454976568 ]LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 614235508204¾§=}÷ò–O OCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚9426007 IDeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4289 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 [HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚69069260816384 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚223 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚2143822 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 5368709120 KDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚59 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 5368709120 E8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚17977262 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 4552982528 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚4287 Q>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚70093642010112 SGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚8555544 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 5368709120 KBDeviceStatsMonitor.on_trêæ]G¥ ain_batch_start/allocation.small_pool.peak‚4876 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚2143750 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 843096576 bQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 651095080960 K?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚9426005 VEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 615476989440 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚59395584 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 $ _timestamp‚1.786659333148441e+09 G>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4876 _LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚69478182125568 SGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚9426007 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚5867273 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚20643840 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚321 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚321 IBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚59 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 122905600 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 IBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚74 UHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚62584832 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 ZIDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 615518934528 JBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚665 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚262 _step‚207 XEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚69478165020672 L@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚8011021 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 5492441088 K?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚8551257 D;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚5264 OCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚9426005 ]JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚61231451537408 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚262 C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚4289 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 4654602240 trainer/global_step‚2549 YHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 615476989440 ^MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 651053139968 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 5492441088 M@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚59049984 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17104896 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚4287 dQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚60580356456448 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 4517085184 YFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚61231388952576 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 123731968 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 123731968 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 123731968 OCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚8551257 TADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚69683454976568 ]LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 614235508204 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17104896 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚41940992 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 XEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚70093701060096 PIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚72 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 4628521206 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚321 F?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚59 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 4552982528 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 122905600 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 IDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 UBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚70093701060096 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚41945088 [HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚69478165020672 ]LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 615518934528 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚429 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 @7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚5264 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚5867271 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 PCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚59049984 PCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚58387380 OCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚8555544 TADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚70093642010112 `MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚60580335812608Ï â+"0ó+ _wandbruntime‚681  _runtime‚681ˆæ¡&ô›M REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚41948160 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 122905600 WFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 615708058112 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚209 [HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚61254744699904 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 S@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚70120254469632 TCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 615708058112 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚321 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 5368709120 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚17988397 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚5869478 \KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 651296835584 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚321 IBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚59 MADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚9429560 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 4552982528 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚262 _runtime‚ 681.83723187 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚4295 MADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚8554544 ^KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚60603383185408 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 WFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 614424732796 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17104896 EDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 % _timestamp‚1.7866593333069553e+09 I=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚9429558 ZGDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚69504504463360 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚2144570 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 YFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚69095446609920 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 5368709120 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚59395584 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 [JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 614466080864 NBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚8014105 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 5492441088 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 5368709120 YFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚69504487358464 ]JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚69095463649280 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 122306526 B5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5264 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚4293 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 SFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚64678912 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 _step‚208 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚5869476 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 R?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚69709871342716 I@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚4876 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚223 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚321 trainer/global_step‚2549 K>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59053056 XGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 615750006272 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 4517085184 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚321 D=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚59 VCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚69504487358464 QEDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚8558837 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚429 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚17988397 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚41348068 VCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚70120254469632 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚4293 ODeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521206 NADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚59053056 ]JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚69504504463360 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚17984102 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚44035072 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 VCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚69709929730144 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 5492441088 G@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚59 WDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚61254680020992 `ODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 651340870656 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 5492441088 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 123731968 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 122905600 NGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚57 bODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚60603403829248 EDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚8014046 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚2144627 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 831258624 NADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚58387428 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚262 G@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚59 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654602240 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17104896 QEDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚9429560 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 DDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654602240 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17104896 QEDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚9429560 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 D8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 S@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚70120254469632 TCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 615708058112 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚321 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 5368709120 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚17988397 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚5869478 \KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 651296835584 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚321 IBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚59 MADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚9429560 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 4552982528 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚262 _runtime‚ 681.83723187 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚4295 MADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚8554544 ^KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚60603383185408 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 WFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 614424732796 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17104896 EDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 % _timestamp‚1.7866593333069553e+09 I=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚9429558 ZGDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚69504504463360 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚2144570 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 YFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚69095446609920 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 5368709120 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚59395584 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 [JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 614466080864 NBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚8014105 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 5492441088 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 5368709120 YFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚69504487358464 ]JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚69095463649280 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 122306526 B5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5264 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚4293 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 SFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚64678912 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 _step‚208 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚5869476 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 R?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚69709871342716 I@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚4876 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚223 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚321 trainer/global_step‚2549 K>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59053056 XGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 615750006272 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 4517085184 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚321 D=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚59 VCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚69504487358464 QEDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚8558837 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚429 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚17988397 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚41348068 VCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚70120254469632 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚4293 ODeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521206 NADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚59053056 ]JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚69504504463360 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚17984102 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚44035072 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 VCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚69709929730144 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 5492441088 G@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚59 WDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚61254680020992 `ODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 651340870656 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 5492441088 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 123731968 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 122905600 NGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚57 bODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚60603403829248 EDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚8014046 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚2144627 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 831258624 NADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚58387428ÐÂÇ50ö+ _wandbruntime‚681  _runtime‚681þ!p÷ê $ _timestamp‚1.786659333307908e+09 /loss/FRt2/assignment_loss‚0.771511261085232 /loss/FRt1/detection_loss‚0.5843875547608453 _runtime‚ 681.837746793 _step‚209 0loss/FRt1/assignment_loss‚0.8749518448266141 0loss/FRt2/detection_loss‚0.47402609051192374 &loss/total_loss‚2.7048767511846155 epoch‚6 trainer/global_step‚2549d±ƒÐuøï 0loss/FRt1/assignment_loss‚0.8749518448266141 0loss/FRt2/detection_loss‚0.47402609051192374 &loss/total_loss‚2.7048767511846155 epoch‚6 trainer/global_step‚2549 $ _timestamp‚1.786659333307908e+09 /loss/FRt2/assignment_loss‚0.771511261085232 /loss/FRt1/detection_loss‚0.5843875547608453 _runtime‚ 681.837746793 _step‚209ÑoB`Š–ù‚(:‹ „ùÓ¤¥–á gpu.0.gpu‚58.0 gpu.0.memory‚11, gpu.0.memoryAllocated‚7.0288848876953125) gpu.0.memoryAllocatedBytes‚ 6037766144 gpu.0.temp‚36.0 gpu.0.powerWatts‚216.266' gpu.0.enforcedPowerLimitWatts‚500.0 gpu.0.powerPercent‚43.2532 gpu.0.smClock‚1410 gpu.0.memoryClock‚1593$ gpu.0.correctedMemoryErrors‚1955# gpu.0.uncorrectedMemoryErrors‚0 ø~ú‚(:ó „ùÓóÁä$ proc.memory.availableMB‚5661.875 cpu‚26.652189665616675 network.recv‚ 1.438638e+06 disk.nvme0c0n1.in‚ 184.171875' disk.nvme0c0n1.out‚5762.54150390625% disk.nvme0n1.out‚5762.54150390625 disk.dm-1.out‚15.0625 disk.drbd2165.in‚0 network.sent‚ 3.426612e+06$ disk./.usageGB‚444.3758239746094 disk.sda.in‚ 5881.49609375 disk.sda2.in‚0 disk.sda2.out‚0 disk.dm-0.in‚ 184.171875* proc.memory.percent‚14.509224891662598 proc.cpu.threads‚17( disk./.usagePercent‚29.8224344030871 disk.dm-1.in‚ 5881.49609375" disk.dm-0.out‚5762.54150390625 disk.sda3.in‚ 5881.49609375$ memory_percent‚65.44265747070312# proc.memory.rssMB‚ 2377.19140625 disk.nvme0n1.in‚ 184.171875 disk.sda.out‚15.0625 disk.sda1.in‚0 disk.sda1.out‚0 disk.drbd2165.out‚0 disk.sda3.out‚15.06258Æí0û+ _wandbruntime‚694  _runtime‚694€KE ÆüÀ & lr-AdamW/pg1‚0.0014538928252119249 & lr-AdamW/pg2‚0.0014538928252119249 trainer/global_step‚2599 % _timestamp‚1.7866593461523333e+09 _runtime‚ 694.682319293 _step‚210°«ÌëËýÅ _step‚210 & lr-AdamW/pg1‚0.0014538928252119249 & lr-AdamW/pg2‚0.0014538928252119249 trainer/global_step‚2599 % _timestamp‚1.7866593461523333e+09 _runtime‚ 694.682319293Òˆæqú0þ+ _wandbruntime‚694  _runtime‚694Âφ´hÿ’O NFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚223 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚321 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚59395584 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 122306526 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 ZIDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 627105700352 [HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚70378550493184 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 5368709120 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 UBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚71421404715520 JBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚665 OCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚8716359 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 4628521206 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚262 _runtime‚ 694.683511966 @7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚5264 YHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 627063755264 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 5492441088 bQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 663388798464 \IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚70794299015168 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocatedÚmÆÔ0 ‚ 5492441088 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 5368709120 VEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 627063755264 _LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚70794299015168 SGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚8720646 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚5977521 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚209 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚262 _step‚211 K?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚8716359 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17104896 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 % _timestamp‚1.7866593461532621e+09 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚20643840 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 5492441088 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 E8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚18320014 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚4287 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 4654602240 XEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚71004364851356 TADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚71004306463976 KBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚4876 UHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚60487680 YFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚62396053407232 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 5368709120 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 F>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚41348020 PCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚59049984 L@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚8161904 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚5977523 ^MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 663348954624 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 SGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚9603657 PCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚58387380 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 4552982528 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚321 TADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚71421345665536 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚41945088 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚429 dQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚61732725096448 _LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚70378567532544 [HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚70794281910272 IBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚70 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚2184451 Q>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚71421345665536 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 PDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚8161974 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚2184383 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 843096576 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚39843840 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚321 ]LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 625797318812 XEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚71421404715520 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 YHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 625755970792 M@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚59049984 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 831258624 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 BDeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4289 IDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4876 XEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚70794281910272 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 4654602240 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 122905600 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 123731968 trainer/global_step‚2599 OCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚9603657 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 122905600 ]LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 627105700352 D;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚5264 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 4517085184 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 123731968 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 ]JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚62396113894912 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚18324303 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 OCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚9603655 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 123731968 IDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚71421345665536 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 PDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚8161974 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚2184383 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 843096576 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚39843840 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚321 ]LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 625797318812 XEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚71421404715520 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 YHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 625755970792 M@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚59049984 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 831258624 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 BDeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4289 IDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4876 XEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚70794281910272 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 4654602240 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 122905600 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 123731968 trainer/global_step‚2599 OCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚9603657 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 122905600 ]LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 627105700352 D;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚5264 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 4517085184 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 123731968 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 ]JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚62396113894912 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚18324303 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 OCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚9603655 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 123731968 IDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 5492441088 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 5368709120 VEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 627063755264 _LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚70794299015168 SGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚8720646 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚5977521 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚209 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚262 _step‚211 K?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚8716359 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17104896 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 % _timestamp‚1.7866593461532621e+09 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚20643840 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 5492441088 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 E8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚18320014 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚4287 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 4654602240 XEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚71004364851356 TADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚71004306463976 KBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚4876 UHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚60487680 YFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚62396053407232 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 5368709120 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 F>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚41348020 PCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚59049984 L@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚8161904 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚5977523 ^MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 663348954624 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0Ó‘ÌÉ0+ _wandbruntime‚694  _runtime‚694Ƚl€¡&‚›M C6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚18326854 YFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚70820604248064 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚209 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 122905600 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚321 K>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59053056 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 WDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚62419341944320 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 4654602240 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17104896 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 R?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚71030722830124 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521206 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 123731968 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚321 $ _timestamp‚1.786659346319403e+09 MADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚8719646 J>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚8164900 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚262 _runtime‚ 694.849920903 _step‚212 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚4295 S@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚71447958125056 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚46132224 IBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚59 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 ]JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚70820621352960 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 4552982528 [HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚62419408720384 I=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚9607208 D8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 G@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚59 NADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚59053056 H@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚665 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚429 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 5492441088 >5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5264 \KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 663590118912 ]JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚70404753326080 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 5368709120 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 TCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 627294823936 QEDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚8723939 I@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚4876 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚2185174 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚18331149 G@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚72 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 YFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚70404736286720 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 5492441088 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 5368709120 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17104896 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚18331149 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚5979728 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 5368709120 VCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚70820604248064 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 122905600 bODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚61755772469248 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 122306526 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚321 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 4552982528 [JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 627336772096 ^KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚61755751825408 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 NADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚58387428 MADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚9607210 EDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚18326854 BDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654602240 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 123731968 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚321 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚4293 VCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚71030781217552 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚262 EDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 R?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚71447899072000 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 [JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 626027891472 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚41948160 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚41948160 MADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚9607208 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚223 SFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚66776064 `ODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 663636251136 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 5492441088 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 D=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚59 I=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚8719646ñLô?ƒ M ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 [JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 626027891472 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚41948160 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚41948160 MADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚9607208 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚223 SFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚66776064 `ODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 663636251136 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 5492441088 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 D=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚59 I=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚8719646 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚18326854 YFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚70820604248064 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚209 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 122905600 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚321 K>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59053056 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 WDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚62419341944320 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 4654602240 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17104896 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 R?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚71030722830124 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521206 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 123731968 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚321 $ _timestamp‚1.786659346319403e+09 MADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚8719646 J>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚8164900 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚262 _runtime‚ 694.849920903 _step‚212 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚4295 S@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚71447958125056 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚46132224 IBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚59 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 ]JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚70820621352960 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 4552982528 [HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚62419408720384 I=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚9607208 D8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 G@DeviceSºFgtatsMonitor.on_train_batch_end/segment.small_pool.current‚59 NADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚59053056 H@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚665 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚429 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 5492441088 >5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5264 \KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 663590118912 ]JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚70404753326080 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 5368709120 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 TCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 627294823936 QEDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚8723939 I@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚4876 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚2185174 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚18331149 G@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚72 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 YFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚70404736286720 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 5492441088 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 5368709120 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17104896 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚18331149 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚5979728 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 5368709120 VCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚70820604248064 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 122905600 bODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚61755772469248 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 122306526 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚321 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 4552982528 [JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 627336772096 ^KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚61755751825408 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 NADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚58387428 MADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚9607210 EDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚18326854 BDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654602240 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 123731968 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚321 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚4293 VCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚71030781217552 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚262 EDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 R?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚71447899072000 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0Ô!ìòÞ0„+ _wandbruntime‚694  _runtime‚694qªÇp…ê 0loss/FRt1/assignment_loss‚0.8821744281552072 /loss/FRt1/detection_loss‚0.5665538172160892 /loss/FRt2/detection_loss‚0.5270404648653138 %loss/total_loss‚2.755355755347702 epoch‚6 trainer/global_step‚2599 0loss/FRt2/assignment_loss‚0.7795870451110918 % _timestamp‚1.7866593463204997e+09 _runtime‚ 694.850344083 _step‚213ÈÒþÑu†ï 0loss/FRt1/assignment_loss‚0.8821744281552072 /loss/FRt1/detection_loss‚0.5665538172160892 /loss/FRt2/detection_loss‚0.5270404648653138 %loss/total_loss‚2.755355755347702 epoch‚6 trainer/global_step‚2599 0loss/FRt2/assignment_loss‚0.7795870451110918 % _timestamp‚1.7866593463204997e+09 _runtime‚ 694.850344083 _step‚213Õz³ÏŽ”‡‚(:‰ œ„ùÓåæ•á gpu.0.gpu‚25.0 gpu.0.memory‚6, gpu.0.memoryAllocated‚7.0288848876953125) gpu.0.memoryAllocatedBytes‚ 6037766144 gpu.0.temp‚36.0 gpu.0.powerWatts‚211.995' gpu.0.enforcedPowerLimitWatts‚500.0 gpu.0.powerPercent‚42.399 gpu.0.smClock‚1410 gpu.0.memoryClock‚1593$ gpu.0.correctedMemoryErrors‚1955# gpu.0.uncorrectedMemoryErrors‚0·]„Žˆ‚(:ƒ œ„ùÓèëã$ disk./.usageGB‚444.3619956970215 disk.dm-1.in‚ 5920.3984375 disk.dm-1.out‚ 15.21484375 disk.drbd2165.in‚0 disk.dm-0.in‚ 187.9921875 disk.sda3.in‚ 5920.3984375 disk.sda3.out‚ 15.21484375 disk.sda.in‚ 5920.3984375 disk.sda.out‚ 15.21484375 disk.sda1.in‚0 disk.sda1.out‚0 disk.sda2.out‚0$ memory_percent‚65.44291973114014 network.recv‚ 1.468766e+06 disk.drbd2165.out‚0! disk.dm-0.out‚6022.8408203125! disk.nvme0c0n1.in‚ 187.9921875 disk.sda2.in‚0 disk.nvme0n1.in‚ 187.9921875$ disk.nvme0n1.out‚6022.8408203125# proc.memory.rssMB‚ 2377.19140625 network.sent‚ 3.499154e+06& disk.nvme0c0n1.out‚6022.8408203125) proc.memory.availableMB‚ 5661.83203125* proc.memory.percent‚14.509224891662598 cpu‚26.662362981616823 proc.cpu.threads‚17* disk./.usagePercent‚29.821506375775474òCè0‰+ _wandbruntime‚706  _runtime‚706±va ÆŠÀ & lr-AdamW/pg2‚0.0014517952609241136 trainer/global_step‚2649 % _timestamp‚1.7866593583323755e+09 _runtime‚ 706.865053763 _step‚214 & lr-AdamW/pg1‚0.0014517952609241136s©ÞOË‹Å & lr-AdamW/pg2‚0.0014517952609241136 trainer/global_step‚2649 % _timestamp‚1.7866593583323755e+09 _runtime‚ 706.865053763 _step‚214 & lr-AdamW/pg1‚0.0014517952609241136ÖBc0Œ+ _wandbruntime‚706  _runtime‚706ß]ßM–'O G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4289 IDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 SGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚8885748 dQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚62885093736448 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 5492441088 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 [HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚71687840169984 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚321 IBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚74 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚429 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚6087773 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚209 bQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 675669492224 TADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚72325157951384 _step‚215 E8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚18662766 YHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 638650521088 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 F>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 5368709120 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 123731968 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 123731968 C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚4289 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚46135296 YHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 637317781400 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 5368709120 F?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚59 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 _LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚71687857209344 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚321 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 5492441088 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 123731968 JBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚665 PCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚59049984 [HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚72110398799872 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚41945088 BDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚72749049320960 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚18667055 PCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚58387380 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 122306526 G>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4876 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 122905600ñÕ%±›'Ž•O OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 OCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚9781307 XEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚72110398799872 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚321 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚262 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 122905600 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚4287 KBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚4876 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚262 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚262 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 `MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚62885073092608 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 5368709120 \IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚72110415904768 L@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚8313216 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 831258624 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 OCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚8881461 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 4628521206 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 $ _timestamp‚1.786659358333028e+09 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 4654602240 D;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚5264 PIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚72 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚59395584 IBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚59 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚2225517 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚6087771 K?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚8881461 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚41945088 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 4552982528 @7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚5264 PDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚8313290 UHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚66779136 YFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚63560696449536 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 M@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚59049984 VEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 638650521088 _LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚72110415904768 SGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚9781307 UBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚72749108370944 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17104896 ]JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚63560763228672 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 5492441088 KDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚59 OCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚9781305 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 843096576 _runtime‚ 706.86516883 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚4287 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚321 ZIDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 638692466176 TADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚72749049320960 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚223 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚2225445 ]LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 637359129420 trainer/global_step‚2649 IDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 SGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚8885748 dQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚62885093736448 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 5492441088 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 [HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚71687840169984 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚321 IBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚74 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚429 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚6087773 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚209 bQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 675669492224 TADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚72325157951384 _step‚215 E8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚18662766 YHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 638650521088 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 F>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 5368709120 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 123731968 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 123731968 C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚4289 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚46135296 YHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 637317781400 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 5368709120 F?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚59 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 _LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚71687857209344 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚321 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 5492441088 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 123731968 JBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚665 PCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚59049984 [HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚72110398799872 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚41945088 BDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚72749049320960 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚18667055 PCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚58387380 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 122306526 G>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4876 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 122905600 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4289 IDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 IBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚59 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 4654602240 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 4552982528 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚321 D=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚59 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 5492441088 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚262 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚2226291 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 5368709120 ZGDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚72136738242560 VCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚72136721137664 TCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 638881589760 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521206 trainer/global_step‚2649 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚18669606 NBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚8316269 J>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚8316195 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚18669606 I=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚8884748 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚41948160 ^KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚62908120465408 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚18673901 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 831258624 YFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚72136721137664 ]JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚72136738242560 I@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚4876 bODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚62908141109248 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚41348068 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚321 _runtime‚ 707.028980369 O8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 YFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚71714025963520 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 5492441088 [JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 638923537920 B9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚5264 WDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚63583985945088 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 F@DeviceStatsMoniNÖ#Çôtor.on_train_batch_end/oversize_allocations.freed‚0 QEDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚8889041 MADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚8884748 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚4293 XGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 638923537920 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17104896 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚48229376 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚41948160 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 4552982528 [HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚63584054818304 \KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 675865479680 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚59395584 VCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚72775661780480 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚262 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17104896 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚429 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚6089976 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 R?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚72351574317532 WFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 637548354012 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚321 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚262 >5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5264 MADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚9784858 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 G@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚59 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 NADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚59053056 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚321 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 123731968 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚4293 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚2226219 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 [JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 637589702080 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚223 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 122306526 WFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 638881589760 R?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚72775602727424 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚18673901 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚4295 EDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59053056 EDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654602240 G@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚74 SFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚68873216 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚209 ]JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚71714043002880 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 123731968 _step‚216 H@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚665 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 122905600 I=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚9784858 DDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521206 trainer/global_step‚2649 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚18669606 NBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚8316269 J>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚8316195 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚18669606 I=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚8884748 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚41948160 ^KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚62908120465408 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚18673901 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 831258624 YFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚72136721137664 ]JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚72136738242560 I@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚4876 bODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚62908141109248 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚41348068 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚321 _runtime‚ 707.028980369 O8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 YFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚71714025963520 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 5492441088 [JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 638923537920 B9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚5264 WDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚63583985945088 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 QEDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚8889041 MADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚8884748 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚4293 XGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 638923537920 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17104896 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚48229376 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚41948160 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 4552982528 [HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚63584054818304 \KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 675865479680 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚59395584 VCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚72775661780480 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚262 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17104896 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚429 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚6089976 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 R?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚72351574317532 WFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 637548354012 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚321 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚262 >5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5264 MADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚9784858 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 G@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚59 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 NADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚59053056 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚321 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 123731968 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚4293 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚2226219 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 [JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 637589702080 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚223 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 122306526 WFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 638881589760 R?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚72775602727424 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚18673901 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚4295 EDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59053056 EDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654602240 G@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚74 SFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚68873216 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚209 ]JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚71714043002880 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 123731968 _step‚216 H@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚665 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 122905600 I=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚9784858 DDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 IBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚59Ø,‰Þ0’+ _wandbruntime‚707  _runtime‚707š5{vp“ê _step‚217 0loss/FRt1/assignment_loss‚0.9656980683740826 /loss/FRt2/detection_loss‚0.5061046016397825 0loss/FRt2/assignment_loss‚0.8076958931883091 /loss/FRt1/detection_loss‚0.6006393936422105 &loss/total_loss‚2.8801379568443846 epoch‚6 trainer/global_step‚2649 % _timestamp‚1.7866593584995916e+09 _runtime‚ 707.02937781Dºu”ï 0loss/FRt2/assignment_loss‚0.8076958931883091 /loss/FRt1/detection_loss‚0.6006393936422105 &loss/total_loss‚2.8801379568443846 epoch‚6 trainer/global_step‚2649 % _timestamp‚1.7866593584995916e+09 _runtime‚ 707.02937781 _step‚217 0loss/FRt1/assignment_loss‚0.9656980683740826 /loss/FRt2/detection_loss‚0.5061046016397825ÙÀ.0•+ _wandbruntime‚719  _runtime‚719( †éÆ–À & lr-AdamW/pg2‚0.0014496526216381045 trainer/global_step‚2699 % _timestamp‚1.7866593713056717e+09 _runtime‚ 719.835798852 _step‚218 & lr-AdamW/pg1‚0.0014496526216381045†M¤ÍË—Å _step‚218 & lr-AdamW/pg1‚0.0014496526216381045 & lr-AdamW/pg2‚0.0014496526216381045 trainer/global_step‚2699 % _timestamp‚1.7866593713056717e+09 _runtime‚ 719.835798852ÚÚ`èþ0˜+ _wandbruntime‚719  _runtime‚719ÇyÛç—'™‘O TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚41945088 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 122905600 TADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚74076752976384 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 5492441088 Q>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚74076752976384 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚41945088 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 122306526 K?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚9958955 XEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚73426515689472 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚429 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 843096576 `MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚64037441732608 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚262 VEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 650237286912 PCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚59049984 KBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚4876 _step‚219 ZIDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 650279232000 [HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚72997129846784 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚41348020 trainer/global_step‚2699 F>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚209 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚321 BDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4876 ]LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 650279232000 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4289 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚2266066 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 123731968 YHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 650237286912 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚262 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 YHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 648879592008 _runtime‚ 719.84097386 ^MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 687901216256 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚321 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 123731968 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 YFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚64725342948864 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚262 SGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚9958957 SGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚9050850 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 5368709120 K?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚9046563 OCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚9958955 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 5368709120 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 OCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚9958957 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 122905600 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚19009807 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚223 PIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚76 UHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚66779136 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚20643840 _LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚72997146886144 @7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚5264 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17104896 OCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚9046563 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚6198023 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 XEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚73646067826172 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 123731968 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚4287 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 4654602240 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 5492441088 F?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚59 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 4654602240 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 4552982528 TADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚73646009438792 _LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚73426532794368 JBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚665 dQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚64037462376448 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 % _timestamp‚1.7866593713067849e+09 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 4552982528 PDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚8464165 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 831258624 PCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚58387380 ]JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚64725409728000 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚321 KDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚59 E8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚19005518 bQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 687947351552 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 4628521206 \IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚73426532794368 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚59395584 UBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚74076812026368 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚46135296 IBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚59 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚6198021 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 5368709120 IDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 5492441088 F?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚59 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 4654602240 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 4552982528 TADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚73646009438792 _LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚73426532794368 JBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚665 dQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚64037462376448 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 % _timestamp‚1.7866593713067849e+09 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 4552982528 PDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚8464165 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 831258624 PCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚58387380 ]JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚64725409728000 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚321 KDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚59 E8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚19005518 bQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 687947351552 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 4628521206 \IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚73426532794368 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚59395584 UBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚74076812026368 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚46135296 IBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚59 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚6198021 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 5368709120 IDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚74076752976384 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚41945088 KEDeviceStatsMonitor.on_train_batch_start/alloca‰tç!tion.large_pool.current‚2 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 122306526 K?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚9958955 XEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚73426515689472 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚429 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 843096576 `MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚64037441732608 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚262 VEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 650237286912 PCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚59049984 KBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚4876 _step‚219 ZIDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 650279232000 [HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚72997129846784 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚41348020 trainer/global_step‚2699 F>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚209 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚321 BDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4876 ]LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 650279232000 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4289 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚2266066 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 123731968 YHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 650237286912 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚262 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 YHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 648879592008 _runtime‚ 719.84097386 ^MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 687901216256 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚321 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 123731968 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 YFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚64725342948864 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚262 SGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚9958957 SGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚9050850 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 5368709120 K?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚9046563 OCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚9958955 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 5368709120 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0Û¶šn€“›‚(:ˆ «„ùÓèò¡á gpu.0.gpu‚0.0 gpu.0.memory‚0, gpu.0.memoryAllocated‚7.0288848876953125) gpu.0.memoryAllocatedBytes‚ 6037766144 gpu.0.temp‚35.0 gpu.0.powerWatts‚216.025' gpu.0.enforcedPowerLimitWatts‚500.0 gpu.0.powerPercent‚43.205 gpu.0.smClock‚1410 gpu.0.memoryClock‚1593$ gpu.0.correctedMemoryErrors‚1955# gpu.0.uncorrectedMemoryErrors‚0Aà{¢~œ‚(:ó «„ùÓãÚŽã network.sent‚ 3.572121e+06 network.recv‚ 1.499294e+06 disk.sda3.in‚ 6847.65625 disk.sda1.in‚0 disk.sda2.in‚0# proc.memory.rssMB‚ 2377.19140625 cpu‚26.67135787144475 proc.cpu.threads‚17* disk./.usagePercent‚29.960460353157682 disk.nvme0c0n1.in‚192.3125 disk.sda.in‚ 6847.65625 disk.sda1.out‚0 disk.drbd2165.in‚0! disk.dm-0.out‚6204.0361328125$ memory_percent‚65.44146537780762% disk./.usageGB‚446.43251037597656 disk.sda2.out‚0$ disk.nvme0n1.out‚6204.0361328125 disk.dm-1.in‚ 6847.65625 disk.dm-1.out‚ 15.33203125 disk.dm-0.in‚192.3125* proc.memory.percent‚14.509224891662598 disk.sda3.out‚ 15.33203125& disk.nvme0c0n1.out‚6204.0361328125 disk.sda.out‚ 15.33203125 disk.nvme0n1.in‚192.3125 disk.drbd2165.out‚0( proc.memory.availableMB‚ 5662.0703125{Ê0+ _wandbruntime‚720  _runtime‚720i§-N¡&ž›M S@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚74103365435904 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 4552982528 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚20643840 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 NADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚58387428 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 122306526 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚19012358 J>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚8467109 SFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚64678912 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 5368709120 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 ZGDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚73452855132160 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17104896 NADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚59053056 I@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚4876 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 ]JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚73452855132160 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚4293 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚321 $ _timestamp‚1.786659371489579e+09 E8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 G@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚59 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 123731968 I=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚9049850 [HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚64748702456320 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 B9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚5264 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚6200228 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 \KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 688148672000 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 [JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 649151512688 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚2266962 K>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59053056 D6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚321 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚19016653 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚41948160 QEDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚9962510 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚59395584 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 4654602240 MADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚9049850 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 5368709120 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚209 trainer/global_step‚2699 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 122905600 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚4295 MADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚9054143 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚41948160 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚44035072 IBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚59 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚4293 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 WFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 649110164620 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 5368709120 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654602240 YFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚73452838027264 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 ]JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚73023332679680 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚262 WDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚64748637777408 BDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521206 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚41348068 I=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚9962508 XGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 650510303744 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 4552982528 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚6200226 `ODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 688192707072 R?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚73672425804940 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 4517085184 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚223 VCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚74103365435904 >5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5264 [JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 650510303744 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚262 G@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚81 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 5492441088 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 123731968 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚321 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 ODeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚19016653 bODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚64060509749248 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 5492441088 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚321 MADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚9962510 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 122905600 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 123731968¸fÜ™¦&Ÿ M M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521206 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚41348068 I=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚9962508 XGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 650510303744 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 4552982528 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚6200226 `ODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 688192707072 R?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚73672425804940 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 4517085184 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚223 VCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚74103365435904 >5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5264 [JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 650510303744 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚262 G@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚81 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 5492441088 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 123731968 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚321 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 ODeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚19016653 bODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚64060509749248 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 5492441088 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚321 MADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚9962510 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 122905600 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 123731968 S@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚74103365435904 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 4552982528 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚20643840 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 NADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚58387428 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 122306526 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚19012358 J>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚8467109 SFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚64678912 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 5368709120 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 ZGDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚73452855132160 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17104896 NADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚59053056 I@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚4876 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 ]JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚73452855132160 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚4293 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚321 $ _timestamp‚1.786659371489579e+09 E8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 G@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚59 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 123731968 I=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚9049850 [HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚64748702456320 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 B9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚5264 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚6200228 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 \KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 688148672000 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 [JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 649151512688 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚2266962 K>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59053056 D6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚321 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚19016653 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚41948160 QEDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚9962510 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚59395584 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 4654602240 MADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚9049850 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 5368709120 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚209 trainer/global_step‚2699 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 122905600 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚4295 MADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚9054143 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚41948160 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚44035072 IBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚59 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚4293 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 WFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 649110164620 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 5368709120 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654602240 YFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚73452838027264 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 ]JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚73023332679680 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚262 WDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚64748637777408 BDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚75404456631808 [HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚74742632579072 `MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚65189810372608 [HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚74306419523584 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 122306526 @7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚5264 M@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚59049984 D;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚5264 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚321 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚10136607 YHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 661824052736 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 843096576 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 YHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 660441402616 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚321 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚19352559 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚10136605 OCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚9211665 ]JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚65890047849472 UHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚64681984 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 123731968 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚262 K?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚9211665 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17104896 _LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚74742649683968 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 PCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚58387380 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 KDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚59 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 4654602240 VEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 661824052736 PCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚59049984 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚44038144 ^MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 700172794880 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚4287 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚262 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 _runtime‚ 732.362911906 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 4654602240 ]LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 661865997824 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚41348020 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚223 _step‚223 E8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚19348270 OCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚9215952 KBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚4876 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚59395584 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚2307059 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚20643840 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 4628521206 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 5492441088 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 123731968 F>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 TADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚75404456631808 IBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚70 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚4287 XEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚74742632579072 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚41945088 IDeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4289 dQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚65189831016448 BDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 5368709120 F?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚59 ]LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 660482750636 JBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚665 YFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚65889983167488 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚262 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 4552982528 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 4552982528 _LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚74306436562944 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 % _timestamp‚1.7866593838293645e+09 G>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4876 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚41945088 PIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚68 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 5492441088 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 5492441088 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 5368709120 trainer/global_step‚2749 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 831258624 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 123731968 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0Žj«¡'¨›O SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 843096576 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 YHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 660441402616 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚321 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚19352559 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚10136605 OCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚9211665 ]JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚65890047849472 UHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚64681984 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 123731968 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚262 K?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚9211665 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17104896 _LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚74742649683968 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 PCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚58387380 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 KDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚59 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 4654602240 VEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 661824052736 PCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚59049984 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚44038144 ^MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 700172794880 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚4287 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚262 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 _runtime‚ 732.362911906 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 4654602240 ]LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 661865997824 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚41348020 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚223 _step‚223 E8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚19348270 OCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚9215952 KBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚4876 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚59395584 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚2307059 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚20643840 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 4628521206 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 5492441088 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 123731968 F>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 TADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚75404456631808 IBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚70 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚4287 XEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚74742632579072 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚41945088 IDeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4289 dQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚65189831016448 BDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 5368709120 F?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚59 ]LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 660482750636 JBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚665 YFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚65889983167488 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚262 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 4552982528 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 4552982528 _LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚74306436562944 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 % _timestamp‚1.7866593838293645e+09 G>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4876 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚41945088 PIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚68 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 5492441088 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 5492441088 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 5368709120 trainer/global_step‚2749 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 831258624 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 123731968 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚6308273 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 L@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚8615330 Q>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚75404456631808 [HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚74742632579072 `MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚65189810372608 [HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚74306419523584 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 122306526 @7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚5264 M@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚59049984 D;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚5264 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚321 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚10136607 YHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 661824052736ß*Æ"10©+ _wandbruntime‚732  _runtime‚732ž“¹¦&ª M H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚4293 ZGDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚74768972021760 TCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 662055121408 VCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚74993335679776 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 5368709120 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17104896 ]JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚74332622356480 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 123731968 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚19359405 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚321 G@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚59 QEDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚9219245 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 YFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚74768954916864 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 5492441088 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 I=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚9214952 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 4654602240 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚19355110 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 843096576 E8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 NADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚58387428 ODeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚8618381 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚6310476 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚19355110 S@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚75431069091328 H@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚665 G@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚66 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 MADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚9214952 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 5368709120 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 123731968 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚321 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚262 VCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚74768954916864 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚59395584 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 D=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚59 IBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚59 % _timestamp‚1.7866593839911819e+09 _runtime‚ 732.521444844 MADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚9219245 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚209 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚41348068 NADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚59053056 NGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚64 `ODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 700460091392 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 123731968 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 122905600 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚20643840 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚321 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654602240 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚6310478 \KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 700418153472 WFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 660671975228 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 122306526 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 5492441088 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚262 _step‚224 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚4295 >5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5264 I@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚4876 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚2307969 BDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521206 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 5492441088 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17104896 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚2307905 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 831258624 K>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59053056 [JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 662097069568 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚10140160 R?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚74993277292348 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚321 XGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 662097069568 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 122905600 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚4293 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 [JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 660713323296 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚10140160 R?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚75431010038272 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 WDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚65913275898880 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚262áÃ#J¹ «¥M G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚19359405 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚41948160 YFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚74332605317120 trainer/global_step‚2749 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚10140158 E8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 NADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚58387428 ODeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚8618381 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚6310476 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚19355110 S@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚75431069091328 H@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚665 G@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚66 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 MADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚9214952 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 5368709120 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 123731968 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚321 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚262 VCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚74768954916864 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚59395584 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 D=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚59 IBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚59 % _timestamp‚1.7866593839911819e+09 _runtime‚ 732.521444844 MADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚9219245 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚209 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚41348068 NADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚59053056 NGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚64 `ODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 700460091392 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 123731968 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 122905600 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚20643840 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚321 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654602240 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚6310478 \KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 700418153472 WFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 660671975228 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 122306526 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 5492441088 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚262 _step‚224 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚4295 >5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5264 I@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚4876 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚2307969 BDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521206 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 5492441088 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17104896 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚2307905 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 831258624 K>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59053056 [JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 662097069568 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚10140160 R?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚74993277292348 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚321 XGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 662097069568 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 122905600 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚4293 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 [JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 660713323296 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚10140160 R?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚75431010038272 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 WDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚65913275898880 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚262 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚4293 ZGDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚74768972021760 TCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 662055121408 VCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚74993335679776 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 5368709120 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17104896 ]JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚74332622356480 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 123731968 K>DevicÓºýòeStatsMonitor.on_train_batch_end/allocation.all.allocated‚19359405 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚321 G@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚59 QEDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚9219245 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 YFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚74768954916864 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 5492441088 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 I=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚9214952 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 4654602240 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚19355110 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 843096576 EDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 XEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚76058749468672 ^MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 712443140608 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 5368709120 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚10314257 ZIDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 673452763648 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚6418523 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 _runtime‚ 744.487916797 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚322 _step‚227 \IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚76058766573568 PIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚70 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 5368709120 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17104896 UHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚68876288 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 4654602240 `MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚66342179012608 bQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 712491373056 trainer/global_step‚2799 IDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4876 IDeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4289 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17104896 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 122905600 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚262 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 PDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚8766515 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚59395584 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 UBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚76732219337216 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 ]JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚67054691029504 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚262 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚41945088 L@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚8766443 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚20643840 ]LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 672044561244 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 122306526 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚10314255 PCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚58387380 _LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚75615726239744 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 5494538240 TADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚76732160287232 YFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚67054622153216 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 OCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚9381054 PCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚59049984 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 845178880 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚41348020 YHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 672003213224 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 [HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚75615709200384 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚322 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚19695311 dQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚66342199656448 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 Q>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚76732160287232 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 4552982528 _LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚76058766573568 IBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚72 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 KDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚60 BDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 4517085184 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 5494538240 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 125829120 ]LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 673452763648¡¾xÔ¡'¶›O TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17104896 UHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚68876288 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 4654602240 `MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚66342179012608 bQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 712491373056 trainer/global_step‚2799 IDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4876 IDeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4289 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17104896 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 122905600 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚262 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 PDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚8766515 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚59395584 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 UBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚76732219337216 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 ]JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚67054691029504 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚262 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚41945088 L@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚8766443 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚20643840 ]LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 672044561244 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 122306526 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚10314255 PCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚58387380 _LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚75615726239744 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 5494538240 TADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚76732160287232 YFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚67054622153216 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 OCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚9381054 PCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚59049984 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 845178880 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚41348020 YHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 672003213224 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 [HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚75615709200384 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚322 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚19695311 dQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚66342199656448 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 Q>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚76732160287232 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 4552982528 _LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚76058766573568 IBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚72 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 KDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚60 BDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 4517085184 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 5494538240 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 125829120 ]LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 673452763648 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚41945088 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚4287 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚429 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚2347922 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚262 F>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 XEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚76058749468672 ^MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 712443140608 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 5368709120 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚10314257 ZIDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 673452763648 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚6418523 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 _runtime‚ 744.487916797 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚322 _step‚227 \IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚76058766573568 PIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚70 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 5368709120ãø›ð›0·+ _wandbruntime‚744  _runtime‚744¶(þ ¸ M MADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚9384347 ^KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚66365226385408 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 5494538240 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚262 _step‚228 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚19697862 >5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5264 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17104896 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚10317810 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 831258624 K>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59053056 H@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚665 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚322 VCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚76314187167184 G@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚60 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 4552982528 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 122905600 J>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚8769453 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚20643840 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 XGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 673683835392 VCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚76758772746752 YFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚76085071806464 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 5494538240 B9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚5264 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521206 G@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚76 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚209 [JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 672275133904 WFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 672233785836 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 5494538240 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 125829120 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 NADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚58387428 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚322 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚2348801 IBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚60 MADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚9380054 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚223 TCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 673641887232 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 5368709120 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚10317808 ODeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654602240 EDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 _runtime‚ 744.666688635 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚41948160 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚41348068 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚19697862 QEDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚9384347 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚2348727 R?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚76314128779756 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 D=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚60 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚10317810 I=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚9380054 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 4654602240 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚19702157 WFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 673641887232 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 5368709120 NADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚59053056 bODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚66365247029248 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 5368709120 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17104896 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚6420726 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚262 trainer/global_step‚2799 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚41937920 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚4293 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚41948160 [JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 673683835392 WDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚67077916457472 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 845178880 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 D6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚322 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 ZGDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚76085088911360 EDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚19702157 ]JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚75641912033280 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 125829120 ]JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚76085088911360 I@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚4876 NBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚8769529 [HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚67077979039232 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚429 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 % _timestamp‚1.7866593961361837e+09 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 SFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚62581760 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚322&ÃIÍ«&¹¥M UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚41348068 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚19697862 QEDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚9384347 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚2348727 R?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚76314128779756 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 D=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚60 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚10317810 I=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚9380054 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 4654602240 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚19702157 WFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 673641887232 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 5368709120 NADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚59053056 bODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚66365247029248 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 5368709120 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17104896 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚6420726 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚262 trainer/global_step‚2799 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚41937920 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚4293 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚41948160 [JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 673683835392 WDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚67077916457472 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 845178880 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 D6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚322 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 ZGDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚76085088911360 EDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚19702157 ]JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚75641912033280 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 125829120 ]JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚76085088911360 I@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚4876 NBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚8769529 [HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚67077979039232 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚429 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 % _timestamp‚1.7866593961361837e+09 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 SFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚62581760 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚322 MADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚9384347 ^KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚66365226385408 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 5494538240 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚262 _step‚228 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚19697862 >5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5264 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17104896 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚10317810 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 831258624 K>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59053056 H@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚665 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚322 VCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚76314187167184 G@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚60 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 4552982528 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 122905600 J>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚8769453 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚20643840 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 XGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 673683835392 VCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚76758772746752 YFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚76085071806464 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 5494538240 B9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚5264 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521206 G@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚76 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚209 [JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 672275133904 WFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 672233785836 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 5494538240 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 125829120 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 NADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚58387428 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚322 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚2348801 IBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚60 MADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚9380054 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚223 TCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 673641887232 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 5368709120 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚10317808 ODeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654602240 EDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 _runtime‚ 744.666688635 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚41948160äâì6q0º+ _wandbruntime‚744  _runtime‚744Þ@&qn»è 0loss/FRt1/assignment_loss‚0.9050763947037269 /loss/FRt2/assignment_loss‚0.765162373205936 %loss/total_loss‚2.763549245089091 % _timestamp‚1.7866593961381056e+09 /loss/FRt1/detection_loss‚0.5945744687574032 .loss/FRt2/detection_loss‚0.498736008422025 epoch‚6 trainer/global_step‚2799 _runtime‚ 744.667960248 _step‚229A!õãs¼í /loss/FRt1/detection_loss‚0.5945744687574032 .loss/FRt2/detection_loss‚0.498736008422025 epoch‚6 trainer/global_step‚2799 _runtime‚ 744.667960248 _step‚229 0loss/FRt1/assignment_loss‚0.9050763947037269 /loss/FRt2/assignment_loss‚0.765162373205936 %loss/total_loss‚2.763549245089091 % _timestamp‚1.7866593961381056e+09å@©¦û•½‚(:Š É„ùÓ‘À®á gpu.0.gpu‚44.0 gpu.0.memory‚5, gpu.0.memoryAllocated‚7.0313262939453125) gpu.0.memoryAllocatedBytes‚ 6039863296 gpu.0.temp‚35.0 gpu.0.powerWatts‚215.272' gpu.0.enforcedPowerLimitWatts‚500.0 gpu.0.powerPercent‚43.0544 gpu.0.smClock‚1410 gpu.0.memoryClock‚1593$ gpu.0.correctedMemoryErrors‚1955# gpu.0.uncorrectedMemoryErrors‚0ûÎ2Š¾‚(:ÿ É„ùÓ¡øãä% disk./.usageGB‚446.48476791381836 disk.sda1.in‚0 disk.sda2.out‚0 disk.dm-0.in‚ 195.0859375 disk.sda3.out‚ 15.6953125 disk.sda.out‚ 15.6953125 cpu‚26.729505194079724 proc.cpu.threads‚17 network.recv‚ 1.56253e+06* disk./.usagePercent‚29.963967400369242 disk.sda2.in‚0# disk.nvme0n1.out‚6731.767578125 disk.dm-1.in‚ 7081.67578125 disk.dm-1.out‚ 15.6953125 disk.drbd2165.out‚0! disk.nvme0c0n1.in‚ 195.0859375 network.sent‚ 3.740333e+06 disk.sda1.out‚0 disk.nvme0n1.in‚ 195.0859375% disk.nvme0c0n1.out‚6731.767578125 disk.sda.in‚ 7081.67578125$ memory_percent‚65.44394493103027( proc.memory.availableMB‚ 5661.6640625# proc.memory.rssMB‚ 2377.19140625 disk.drbd2165.in‚0 disk.dm-0.out‚6731.767578125 disk.sda3.in‚ 7081.67578125* proc.memory.percent‚14.509224891662598®£ô0¿+ _wandbruntime‚750  _runtime‚750À•!ÔÀÎ (loss/val_total_loss‚2.81652604308129 /Purity/*FRt/event_purity‚0.4004421830177307 Purity/0FRt/event_purity‚1 % _timestamp‚1.7866594015201423e+09 -Purity/*FRt/FRt_purity‚0.4128913879394531 -Purity/1FRt/FRt_purity‚0.4050171375274658 *jet/accuracy_1_of_1‚0.2693830728530884 )jet/accuracy_1_of_2‚0.508211076259613 particle/accuracy_1_of_1‚1 $particle/accuracy‚0.614990234375 _runtime‚ 750.050197225 _step‚230 3Purity/0FRt/event_proportion‚0.3301943838596344 +particle/sensitivity‚0.6418106555938721 7validation_average_jet_accuracy‚0.27337250113487244 -Purity/2FRt/FRt_purity‚0.4744376242160797 2Purity/*FRt/event_proportion‚0.669805645942688 2Purity/2FRt/event_proportion‚0.040062315762043 +validation_accuracy‚0.16429372131824493 epoch‚6 3Purity/1FRt/event_proportion‚0.6297432780265808 /Purity/2FRt/event_purity‚0.3277498185634613 +jet/accuracy_2_of_2‚0.16429372131824493 /particle/accuracy_1_of_2‚0.8973801136016846 /particle/accuracy_2_of_2‚0.7015660405158997 /Purity/1FRt/event_purity‚0.4050171375274658 +particle/specificity‚0.6002557873725891 'particle/f_score‚0.5419572591781616 trainer/global_step‚2799GΠWÙÁÓ +validation_accuracy‚0.16429372131824493 epoch‚6 3Purity/1FRt/event_proportion‚0.6297432780265808 /Purity/2FRt/event_purity‚0.3277498185634613 +jet/accuracy_2_of_2‚0.16429372131824493 /particle/accuracy_1_of_2‚0.8973801136016846 /particle/accuracy_2_of_2‚0.7015660405158997 /Purity/1FRt/event_purity‚0.4050171375274658 +particle/specificity‚0.6002557873725891 'particle/f_score‚0.5419572591781616 trainer/global_step‚2799 (loss/val_total_loss‚2.81652604308129 /Purity/*FRt/event_purity‚0.4004421830177307 Purity/0FRt/event_purity‚1 % _timestamp‚1.7866594015201423e+09 -Purity/*FRt/FRt_purity‚0.4128913879394531 -Purity/1FRt/FRt_purity‚0.4050171375274658 *jet/accuracy_1_of_1‚0.2693830728530884 )jet/accuracy_1_of_2‚0.508211076259613 particle/accuracy_1_of_1‚1 $particle/accuracy‚0.614990234375 _runtime‚ 750.050197225 _step‚230 3Purity/0FRt/event_proportion‚0.3301943838596344 +particle/sensitivity‚0.6418106555938721 7validation_average_jet_accuracy‚0.27337250113487244 -Purity/2FRt/FRt_purity‚0.4744376242160797 2Purity/*FRt/event_proportion‚0.669805645942688 2Purity/2FRt/event_proportion‚0.040062315762043æÅîFG0Â+ _wandbruntime‚763  _runtime‚763{¬Äþ & lr-AdamW/pg1‚0.0014429556443902615 & lr-AdamW/pg2‚0.0014429556443902615 trainer/global_step‚2849 $ _timestamp‚1.786659415140952e+09 _runtime‚ 763.67290649 _step‚231Z4sÉÄà & lr-AdamW/pg1‚0.0014429556443902615 & lr-AdamW/pg2‚0.0014429556443902615 trainer/global_step‚2849 $ _timestamp‚1.786659415140952e+09 _runtime‚ 763.67290649 _step‚231ç¿ØX‚0Å+ _wandbruntime‚763  _runtime‚763Ò($›'Æ•O MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 JBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚665 dQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚67793838604288 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 XEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚77765803573248 TADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚78454868883968 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 122905600 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 PCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚58387380 C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚4289 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚4287 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 4552982528 IBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚79 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚41348020 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚262 trainer/global_step‚2849 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 122905600 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 4654602240 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 831258624 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 OCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚9564061 IDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚78454868883968 _LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚77765820678144 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚6569387 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚10552513 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17104896 PIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚77 F?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚60 K?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚9564061 D;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚5264 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚223 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 UBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚78454927933952 `MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚67793817960448 XEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚77993587752342 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 4517085184 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚322 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 122306526 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 5494538240 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚322 KDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚60 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚2397857 UHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚66779136 [HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚77305898369024 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 125829120 M@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚59049984 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚10552511 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 F>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚429 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚262 G>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4876 \IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚77765820678144 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 5368709120 IBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚60 _runtime‚ 763.673021898 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚322 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚20120861 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚6569385 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 L@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚8967242 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚10552513 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 YHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 687630995938 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 125829120 _step‚232 PCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚59049984 SGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚9568348 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚262 XEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚78454927933952 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4289 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚10552511 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚4287 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚20643840 ^MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 728808557568 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚41945088 bQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 728854692864 _LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚77305915408384 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 5368709120 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 5494538240 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚322 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 125829120 VEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 689065310720 YHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 689065310720 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚2397934 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 5494538240 $ _timestamp‚1.786659415141653e+09ô·ÿú¾ÇšO OCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚9564061 IDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚78454868883968 _LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚77765820678144 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚6569387 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚10552513 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17104896 PIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚77 F?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚60 K?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚9564061 D;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚5264 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚223 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 UBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚78454927933952 `MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚67793817960448 XEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚77993587752342 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 4517085184 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚322 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 122306526 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 5494538240 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚322 KDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚60 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚2397857 UHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚66779136 [HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚77305898369024 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 125829120 M@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚59049984 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚10552511 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 F>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚429 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚262 G>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4876 \IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚77765820678144 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 5368709120 IBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚60 _runtime‚ 763.673021898 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚322 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 M@DeviceStatsMonitor.on_train_batch6têvâ_start/allocation.all.allocated‚20120861 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚6569385 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 L@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚8967242 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚10552513 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 YHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 687630995938 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 125829120 _step‚232 PCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚59049984 SGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚9568348 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚262 XEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚78454927933952 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4289 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚10552511 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚4287 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚20643840 ^MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 728808557568 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚41945088 bQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 728854692864 _LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚77305915408384 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 5368709120 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 5494538240 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚322 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 125829120 VEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 689065310720 YHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 689065310720 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚2397934 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 5494538240 $ _timestamp‚1.786659415141653e+09 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 JBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚665 dQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚67793838604288 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 XEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚77765803573248 TADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚78454868883968 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 122905600 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 PCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚58387380 C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚4289 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚4287 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 4552982528 IBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚79 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚41348020 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚262 trainer/global_step‚2849 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 122905600 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 4654602240 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 831258624 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0襯žh0È+ _wandbruntime‚763  _runtime‚7631á×Ħ&É M G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚20123412 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 NGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚70 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚59395584 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚322 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17104896 VCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚77792125911040 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 DDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 ZGDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚77792143015936 NADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚59053056 [JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 689338327552 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚10556064 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚20123412 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚10556066 G@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚72 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 5494538240 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚322 R?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚78481422290432 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚20127707 ^KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚67816865333248 B8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚41348068 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚2398686 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 5494538240 _runtime‚ 763.837832765 ODeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59053056 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17104896 \KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 729053588480 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚10556064 [HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚68545985697792 YFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚77332084162560 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654602240 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚4293 bODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚67816885977088 R?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚78019945731110 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 122306526 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 WDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚68545918921728 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521206 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 _step‚233 SFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚66776064 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 831258624 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 5494538240 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 5368709120 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚262 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 4552982528 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚262 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚41948160 J>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚8970276 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚4295 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚429 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 I@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚4876 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚6571592 NADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚58387428 G@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚60 EDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚20643840 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 ]JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚77332101201920 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 5368709120 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 125829120 VCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚78481481343488 NBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚8970348 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚209 `ODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 729099720704 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 4517085184 IBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚60 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 122905600 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚4293 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚2398756 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚322 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚223 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 VCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚78020004118538 >5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5264 XGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 689338327552 TCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 689296379392 ]JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚77792143015936 WFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 6892963793927ñ]«&Ê¥M =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 _step‚233 SFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚66776064 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 831258624 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 5494538240 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 5368709120 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚262 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 4552982528 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚262 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚41948160 J>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚8970276 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚4295 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚429 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 I@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚4876 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚6571592 NADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚58387428 G@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚60 EDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚20643840 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 ]JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚77332101201920 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 5368709120 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 125829120 VCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚78481481343488 NBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚8970348 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚209 `ODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 729099720704 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 4517085184 IBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚60 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 122905600 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚4293 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚2398756 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚322 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚223 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 VCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚78020004118538 >5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5264 XGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 689338327552 TCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 689296379392 ]JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚77792143015936 WFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 689296379392 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚20123412 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 NGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚70 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚59395584 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚322 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17104896 VCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚77792125911040 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 DDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 ZGDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚77792143015936 NADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚59053056 [JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 689338327552 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚10556064 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚20123412 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚10556066 G@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚72 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 5494538240 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚322 R?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚78481422290432 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚20127707 ^KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚67816865333248 B8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚41348068 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚2398686 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 5494538240 _runtime‚ 763.837832765 ODeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59053056 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17104896 \KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 729053588480 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚10556064 [HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚68545985697792 YFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚77332084162560 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654602240 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚4293 bODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚67816885977088 R?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚78019945731110 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 122306526 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 WDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚68545918921728 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521206é ²Ó0Ë+ _wandbruntime‚763  _runtime‚76355üoÌé 0loss/FRt2/assignment_loss‚0.7687638069344201 %loss/total_loss‚2.684566274338854 epoch‚7 trainer/global_step‚2849 _runtime‚ 763.838268167 _step‚234 0loss/FRt1/assignment_loss‚0.8539988135308536 /loss/FRt1/detection_loss‚0.5427934988150388 /loss/FRt2/detection_loss‚0.5190101550585415 $ _timestamp‚1.786659415308439e+09ïxhÎtÍî 0loss/FRt1/assignment_loss‚0.8539988135308536 /loss/FRt1/detection_loss‚0.5427934988150388 /loss/FRt2/detection_loss‚0.5190101550585415 $ _timestamp‚1.786659415308439e+09 0loss/FRt2/assignment_loss‚0.7687638069344201 %loss/total_loss‚2.684566274338854 epoch‚7 trainer/global_step‚2849 _runtime‚ 763.838268167 _step‚234ê­Æҕ΂(:Š Ø„ùÓ©¡µá gpu.0.gpu‚20.0 gpu.0.memory‚5, gpu.0.memoryAllocated‚7.0313262939453125) gpu.0.memoryAllocatedBytes‚ 6039863296 gpu.0.temp‚33.0 gpu.0.powerWatts‚119.454' gpu.0.enforcedPowerLimitWatts‚500.0 gpu.0.powerPercent‚23.8908 gpu.0.smClock‚1410 gpu.0.memoryClock‚1593$ gpu.0.correctedMemoryErrors‚1955# gpu.0.uncorrectedMemoryErrors‚0<¥Æ‹Ï‚(:€ Ø„ùӵǦã proc.cpu.threads‚17 network.recv‚ 1.595201e+06 disk.drbd2165.in‚0 disk.dm-0.in‚ 232.9609375! disk.dm-0.out‚6766.0068359375 disk.sda3.in‚ 7081.67578125& disk.nvme0c0n1.out‚6766.0068359375 disk.sda1.out‚0 disk.sda2.in‚0 network.sent‚ 3.816306e+06 disk.sda3.out‚ 15.95703125 disk.sda.in‚ 7081.67578125 disk.sda1.in‚0 disk.nvme0n1.in‚ 232.9609375$ memory_percent‚65.06309509277344* proc.memory.percent‚14.128971099853516 cpu‚26.737471169173254 disk.dm-1.in‚ 7081.67578125! disk.nvme0c0n1.in‚ 232.9609375 disk.sda2.out‚0% proc.memory.availableMB‚ 5724.0625! proc.memory.rssMB‚ 2314.890625* disk./.usagePercent‚29.963973032534994$ disk./.usageGB‚446.4848518371582 disk.dm-1.out‚ 15.95703125 disk.drbd2165.out‚0 disk.sda.out‚ 15.95703125$ disk.nvme0n1.out‚6766.0068359375(" †0Ð+ _wandbruntime‚776  _runtime‚776Ó»§`ÆÑÀ & lr-AdamW/pg2‚0.0014406341101057699 trainer/global_step‚2899 % _timestamp‚1.7866594280812273e+09 _runtime‚ 776.611314012 _step‚235 & lr-AdamW/pg1‚0.0014406341101057699 3JƒËÒÅ & lr-AdamW/pg2‚0.0014406341101057699 trainer/global_step‚2899 % _timestamp‚1.7866594280812273e+09 _runtime‚ 776.611314012 _step‚235 & lr-AdamW/pg1‚0.0014406341101057699ë‡?†=0Ó+ _wandbruntime‚776  _runtime‚776âËh@Ô–O XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 5368709120 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚44038144 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 KDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚60 _step‚236 YHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 700652076544 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚6679637 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚2438640 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 125829120 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚2438569 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 125829120 Q>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚79782572539392 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚6679635 dQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚68946207244288 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 122306526 F>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 ZIDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 700694021632 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚59395584 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 TADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚79314380852370 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 \IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚79081937567744 JBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚665 BDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚322 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚10730161 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 5494538240 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚262 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚41945088 _runtime‚ 776.615924372 XEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚79782631589376 TADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚79782572539392 [HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚78615188045824 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚41348020 PDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚9118277 IDeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4289 OCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚9729163 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚322 ]LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 699234154566 % _timestamp‚1.7866594280823007e+09 G>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4876 KBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚4876 IBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚73 PIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚71 ^MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 741085703680 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 4628521206 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 @7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚5264 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 4552982528 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚10730163 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚322 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 122905600 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 122905600 `MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚68946186600448 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 trainer/global_step‚2899 E8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚20459324 _LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚79081937567744 [HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚79081920462848 C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚4289 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 4654602240 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚322 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚10730161 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 4552982528 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 845178880 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 5368709120 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17104896 IDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚79782572539392 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚6679635 dQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚68946207244288 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 122306526 F>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 ZIDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 700694021632 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚59395584 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 TADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚79314380852370 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 \IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚79081937567744 JBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚665 BDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚322 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚10730161 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 5494538240 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚262 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚41945088 _runtime‚ 776.615924372 XEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚79782631589376 TADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚79782572539392 [HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚78615188045824 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚41348020 PDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚9118277 IDeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4289 OCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚9729163 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚322 ]LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 699234154566 % _timestamp‚1.7866594280823007e+09 G>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4876 KBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚4876 IBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚73 PIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚71 ^MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 741085703680 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 4628521206 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 @7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚5264 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 4552982528 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚10730163 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚322 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 122905600 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 122905600 `MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚68946186600448 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 trainer/global_step‚2899 E8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚20459324 _LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚79081937567744 [HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚79081920462848 C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚4289 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 4654602240 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚322 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚10730161 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 4552982528 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 845178880 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 5368709120 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17104896 IDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 ]JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚78641390878720 [JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 699464727226 G@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚60 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚10733714 NADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚58387428 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 125829120 NGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚65 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 125829120 XGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 700925093376 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚209 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 4517085184 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚262 IBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚60 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚10733716 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚4295 VCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚79108242800640 I@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚4876 `ODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 741373884928 WFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 699423379158 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚262 _step‚237 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 4552982528 K>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59053056 R?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚79809125945856 J>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚9121242 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚2439402 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 845178880 B8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 E6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚322 trainer/global_step‚2899 _runtime‚ 776.794205515 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚4293 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654602240 YFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚79108242800640 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚41948160 B9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚5264 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 [HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚69710628502016 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚59395584 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚20470459 H@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚665 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 5494538240 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 4552982528 QEDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚9736743 \KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 741334044160 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521206 % _timestamp‚1.7866594282637877e+09 TCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 700883145216 G@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚67 ODeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚20470459 MADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚9732450 DDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 5368709120 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚41948160 ]JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚79108259905536 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚10733714 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚41348068 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 5494538240 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚322 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17104896 WFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 700883145216 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 122905600 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚223 R?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚79340797218518 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 122905600 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚322 >5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5264 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚10733716 NBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚9121309 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17104896 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 5368709120 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 SFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚60484608 VCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚79340855605946 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚20466164 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚20466164 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚6681840 WDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚69710568017408 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0NNè•á Ø¥M K>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59053056 R?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚79809125945856 J>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚9121242 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚2439402 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 845178880 B8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 E6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚322 trainer/global_step‚2899 _runtime‚ 776.794205515 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚4293 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654602240 YFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚79108242800640 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚41948160 B9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚5264 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 [HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚69710628502016 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚59395584 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚20470459 H@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚665 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 5494538240 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 4552982528 QEDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚9736743 \KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 741334044160 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521206 % _timestamp‚1.7866594282637877e+09 TCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 700883145216 G@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚67 ODeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚20470459 MADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚9732450 DDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 5368709120 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚41948160 ]JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚79108259905536 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚10733714 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚41348068 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 5494538240 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚322 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17104896 WFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 700883145216 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 122905600 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚223 R?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚79340797218518 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 122905600 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚322 >5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5264 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚10733716 NBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚9121309 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17104896 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 5368709120 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 SFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚60484608 VCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚79340855605946 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚20466164 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚20466164 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚6681840 WDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚69710568017408 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚262 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 ]JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚78641390878720 [JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 699464727226 G@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚60 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚10733714 NADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚58387428 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 125829120 NGDeviceStatsMonitor.on_train_batch_end/inacti&kýÊve_split.small_pool.current‚65 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 125829120 XGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 700925093376 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚209 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 4517085184 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚262 IBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚60 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚10733716 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚4295 VCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚79108242800640 I@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚4876 `ODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 741373884928 WFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 699423379158 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚262 _step‚237 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 4552982528íç~^0Ù+ _wandbruntime‚776  _runtime‚776H apÚê % _timestamp‚1.7866594282652116e+09 _step‚238 /loss/FRt1/detection_loss‚0.5229597489041113 %loss/total_loss‚2.739904619349323 trainer/global_step‚2899 _runtime‚ 776.795108501 0loss/FRt1/assignment_loss‚0.8895857044703758 0loss/FRt2/assignment_loss‚0.8126820120492763 /loss/FRt2/detection_loss‚0.5146771539255599 epoch‚79ãeÓuÛï 0loss/FRt1/assignment_loss‚0.8895857044703758 0loss/FRt2/assignment_loss‚0.8126820120492763 /loss/FRt2/detection_loss‚0.5146771539255599 epoch‚7 % _timestamp‚1.7866594282652116e+09 _step‚238 /loss/FRt1/detection_loss‚0.5229597489041113 %loss/total_loss‚2.739904619349323 trainer/global_step‚2899 _runtime‚ 776.795108501)„–Ü‚(:‹ ç„ùÓÔÊá gpu.0.gpu‚59.0 gpu.0.memory‚11, gpu.0.memoryAllocated‚7.0313262939453125) gpu.0.memoryAllocatedBytes‚ 6039863296 gpu.0.temp‚35.0 gpu.0.powerWatts‚216.832' gpu.0.enforcedPowerLimitWatts‚500.0 gpu.0.powerPercent‚43.3664 gpu.0.smClock‚1410 gpu.0.memoryClock‚1593$ gpu.0.correctedMemoryErrors‚1955# gpu.0.uncorrectedMemoryErrors‚0¬È—K~Ý‚(:ó ç„ùÓ§‘èã disk.dm-1.out‚ 16.3515625 disk.sda3.in‚ 7081.67578125# memory_percent‚65.0632381439209( proc.memory.availableMB‚ 5724.0390625 disk.nvme0n1.out‚ 7077.265625 disk.sda.in‚ 7081.67578125 disk.sda.out‚ 16.3515625 disk.sda2.in‚0 disk.drbd2165.out‚0 cpu‚26.743785803330766 proc.cpu.threads‚17 disk.dm-1.in‚ 7081.67578125 disk.drbd2165.in‚0 disk.sda3.out‚ 16.3515625! proc.memory.rssMB‚ 2314.890625* proc.memory.percent‚14.128971099853516 disk.nvme0n1.in‚ 232.9609375 network.sent‚ 3.890821e+06* disk./.usagePercent‚29.963958696113075$ disk./.usageGB‚446.4846382141113! disk.nvme0c0n1.in‚ 232.9609375 disk.sda1.out‚0 disk.dm-0.in‚ 232.9609375 disk.dm-0.out‚ 7077.265625 network.recv‚ 1.627658e+06" disk.nvme0c0n1.out‚ 7077.265625 disk.sda1.in‚0 disk.sda2.out‚0,¯gÿ0Þ+ _wandbruntime‚789  _runtime‚789T¬¢HÄß¾ % _timestamp‚1.7866594408581073e+09 _runtime‚ 789.388156462 _step‚239 % lr-AdamW/pg1‚0.001438268217682941 % lr-AdamW/pg2‚0.001438268217682941 trainer/global_step‚2949SYŽÉàà _step‚239 % lr-AdamW/pg1‚0.001438268217682941 % lr-AdamW/pg2‚0.001438268217682941 trainer/global_step‚2949 % _timestamp‚1.7866594408581073e+09 _runtime‚ 789.388156462ïÃ5=¤0á+ _wandbruntime‚789  _runtime‚789™ÿÊœ'â–O NFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚223 ZIDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 712280787456 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 122905600 F>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 4654602240 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚209 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 125829120 \IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚80398054457344 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚10907811 PIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚69 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚41348020 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚4287 K?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚9894265 L@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚9269471 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚262 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚262 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 845178880 `MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚70098555240448 ^MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 753358694400 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 IBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚60 G>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4876 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 4552982528 bQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 753402732544 BDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚81110276194816 XEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚80635290727158 _step‚240 OCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚9898552 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 4628521206 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚10907813 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 5368709120 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 122905600 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚10907813 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 PCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚58387380 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 5494538240 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚4289 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 PCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚59049984 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚20806365 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚6789887 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 5494538240 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 UBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚81110335244800 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17104896 KBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚4876 ]JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚70851978616832 _LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚79924494761984 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚322 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 YFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚70851913934848 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 125829120 UHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚64681984 dQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚70098575884288 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 122306526 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 125829120 % _timestamp‚1.7866594408591065e+09 XEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚80398037352448 IBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚71 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚322 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚6789885 E8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚20802076 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚10907811 ]LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 712280787456 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4289 IDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 UBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚81110335244800 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17104896 KBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚4876 ]JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚70851978616832 _LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚79924494761984 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚322 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 YFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚70851913934848 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 125829120 UHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚64681984 dQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚70098575884288 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 122306526 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 125829120 % _timestamp‚1.7866594408591065e+09 XEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚80398037352448 IBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚71 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚322 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚6789885 E8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚20802076 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚10907811 ]LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 712280787456 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4289 IDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 4654602240 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚209 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 125829120 \IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚80398054457344 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚10907811 PIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚69 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚41348020 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚4287 K?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚9894265 L@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚9269471 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚262 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚262 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 845178880 `MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚70098555240448 ^MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 753358694400 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 IBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚60 G>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4876 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 4552982528 bQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 753402732544 BDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚81110276194816 XEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚80635290727158 _step‚240 OCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚9898552 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 4628521206ðsѳ0ä+ _wandbruntime‚789  _runtime‚789Ñ\HQ åŸM WDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚70875208075264 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚20813211 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 4654602240 WFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 712469911040 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚6792090 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 IBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚60 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 VCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚81136888654336 R?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚81136829601280 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚429 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚41348068 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 122306526 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 5494538240 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚262 _step‚241 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚10911366 ]JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚80424376795136 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 ZGDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚80424376795136 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚10911364 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521206 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚6792092 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚209 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚41937920 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 _runtime‚ 789.572246328 S@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚81136888654336 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 4552982528 TCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 712469911040 I@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚4876 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚2480452 [HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚70875270657024 YFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚79950663516160 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 5368709120 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 122905600 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚262 trainer/global_step‚2949 O5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5264 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 MADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚9897552 SFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚62581760 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 XGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 712511859200 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654602240 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17104896 [JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 712511859200 ^KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚70121602613248 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 125829120 DDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59053056 E8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 J>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚9272480 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 831258624 R?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚80661648705926 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 4517085184 D=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚60 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 WFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 710985189766 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17104896 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 4552982528 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚4295 VCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚80424359690240 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 122905600 NGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚62 bODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚70121623257088 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 5368709120 >6DeviceStatsMonitor.on_train_batch_VÎøþTend/segment.all.peak‚322 EDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚20813211 B6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚322 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 [JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 711026537834Ԕػª&æ¤M HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚20813211 B6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚322 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 [JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 711026537834 WDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚70875208075264 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚20813211 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 4654602240 WFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 712469911040 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚6792090 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 IBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚60 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 VCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚81136888654336 R?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚81136829601280 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚429 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚41348068 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 122306526 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 5494538240 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚262 _step‚241 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚10911366 ]JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚80424376795136 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 ZGDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚80424376795136 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚10911364 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521206 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚6792092 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚209 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚41937920 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 _runtime‚ 789.572246328 S@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚81136888654336 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 4552982528 TCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 712469911040 I@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚4876 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚2480452 [HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚70875270657024 YFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚79950663516160 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 5368709120 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 122905600 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚262 trainer/global_step‚2949 O5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5264 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 MADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚9897552 SFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚62581760 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 XGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 712511859200 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654602240 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17104896 [JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 712511859200 ^KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚70121602613248 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 125829120 DDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59053056 E8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 J>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚9272480 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 831258624 R?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚80661648705926 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 4517085184 D=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚60 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 WFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 710985189766 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17104896 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 4552982528 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚4295 VCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚80424359690240 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 122905600 NGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚62 bODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚70121623257088 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 5368709120 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚322 EDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4876 \IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚81714171346944 F?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚60 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚41945088 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 PDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚9420812 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 845178880 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 L@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚9420745 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 _LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚81233784438784 KDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚60 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 M@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚59049984 Q>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚82437979850240 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚10059367 IDeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4289 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚11085461 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚10059367 `MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚71250923880448 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚4287 UHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚60487680 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚20643840 BDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 [HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚81233767399424 IBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚60 IDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17104896 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚41945088 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚2520675 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 @7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚5264 JBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚665 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 5368709120 TADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚82437979850240 PIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚65 ^MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 765650798080 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 VEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 723825608192 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚10063654 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 122306526 trainer/global_step‚2999 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 5494538240 _LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚81714171346944 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 122905600 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚39843840 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚6900135 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 XEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚81714154242048 ]LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 723867553280 KBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚4876 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 5494538240 IBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚67 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚21149117 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚41348020 _runtime‚ 801.838928324 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 5368709120 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚262 _step‚244¸z5à ñžO dQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚71250944524288 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚322 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚11085461 YHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 723825608192 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 4517085184 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚322 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 5368709120 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚6900137 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚11085463 F>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17104896 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚41945088 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚2520675 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 @7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚5264 JBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚665 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 5368709120 TADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚82437979850240 PIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚65 ^MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 765650798080 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 VEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 723825608192 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚10063654 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 122306526 trainer/global_step‚2999 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 5494538240 _LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚81714171346944 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 122905600 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚39843840 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚6900135 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 XEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚81714154242048 ]LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 723867553280 KBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚4876 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 5494538240 IBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚67 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚21149117 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚41348020 _runtime‚ 801.838928324 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 5368709120 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚262 _step‚244 UBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚82438038900224 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚223 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 ]LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 722357775782 $ _timestamp‚1.786659453305479e+09 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 831258624 bQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 765690641920 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚262 G>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4876 \IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚81714171346944 F?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚60 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚41945088 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 PDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚9420812 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 845178880 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 L@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚9420745 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 _LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚81233784438784 KDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚60 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 M@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚59049984 Q>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚82437979850240 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚10059367 IDeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4289 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚11085461 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚10059367 `MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚71250923880448 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚4287 UHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚60487680 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚20643840 BDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 [HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚81233767399424 IBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚60 I6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚322 B9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚5264 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚59395584 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚41348068 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 5494538240 XGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 724098625024 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17104896 WDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚72039866795520 bODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚71273991897088 `ODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 765941674496 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 5368709120 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚322 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 122905600 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚21151668 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚6902340 R?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚81982500193334 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521206 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚262 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 5368709120 DDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚9423755 H@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚665 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 125829120 D=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚60 R?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚82464533256704 YFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚81740476579840 I@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚4876 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚2521489 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 125829120 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚10066947 NADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚59053056 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚322 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 IBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚60 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚10062654 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚21155963 NBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚9423831 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚6902342 WFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 722547000374 SFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚66776064 G@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚60 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚429 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚21155963 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 4654602240 ZGDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚81740493684736 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17104896 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 4552982528 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 5494538240 K>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59053056 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚11089016 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 _step‚245 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚209 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 125829120 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚322 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654602240 S@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚82464592309760 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚223 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚262 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚41948160 G@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚76 ^KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚71273971253248 YFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚81259953192960 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 4517085184 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚4293 [JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 724098625024 WFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 724056676864 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚46132224 VCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚81982558580762 [JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 722588348442 >5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5264 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚11089014 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 845178880 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚4295 EDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 _runtime‚ 802.025661233 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚11089014 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚10062654 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚2521415 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚10066947 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚262 % _timestamp‚1.7866594534951465e+09 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 VCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚82464592309760 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 122905600 NADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚58387428 ODeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚9423755 H@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚665 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 125829120 D=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚60 R?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚82464533256704 YFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚81740476579840 I@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚4876 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚2521489 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 125829120 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚10066947 NADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚59053056 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚322 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 IBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚60 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚10062654 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚21155963 NBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚9423831 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚6902342 WFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 722547000374 SFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚66776064 G@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚60 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚429 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚21155963 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 4654602240 ZGDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚81740493684736 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17104896 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 4552982528 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 5494538240 K>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59053056 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚11089016 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 _step‚245 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚209 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 125829120 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚322 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654602240 S@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚82464592309760 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚223 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚262 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚41948160 G@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚76 ^KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚71273971253248 YFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚81259953192960 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 4517085184 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚4293 [JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 724098625024 WFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 724056676864 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚46132224 VCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚81982558580762 [JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 722588348442 >5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5264 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚11089014 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 845178880 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚4295 EDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 _runtime‚ 802.025661233 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚11089014 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚10062654 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚2521415 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚10066947 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚262 % _timestamp‚1.7866594534951465e+09 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 VCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚82464592309760 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 122905600 NADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚58387428 O6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚322 B9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚5264 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚59395584 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚41348068 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 5494538240 XGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 724098625024 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17104896 WDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚72039866795520 bODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚71273991897088 `ODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 765941674496 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 5368709120 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚322 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 122905600 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚21151668 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚6902340 R?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚81982500193334 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521206 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚262 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 5368709120 DDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4876 M@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚59049984 XEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚83030271131648 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17104896 JBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚665 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚209 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 5494538240 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚10228756 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚4287 dQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚72403313164288 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚322 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 4628521206 F>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 VEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 735412374016 BDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚83765683505664 _LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚83030288236544 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚10224469 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 122306526 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚11263113 YHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 735412374016 YFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚73181230147072 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 KDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚60 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 125829120 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚4287 TADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚83765683505664 L@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚9571635 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 TADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚83276935314594 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 5494538240 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 5368709120 IDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚4289 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4289 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚262 D;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚5264 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚59395584 IDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4876 M@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚59049984 XEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚83030271131648 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17104896 JBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚665 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚209 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 5494538240 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚10228756 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚4287 dQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚72403313164288 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚322 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 4628521206 F>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 VEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 735412374016 BDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚83765683505664 _LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚83030288236544 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚10224469 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 122306526 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚11263113 YHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 735412374016 YFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚73181230147072 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 KDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚60 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 125829120 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚4287 TADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚83765683505664 L@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚9571635 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 TADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚83276935314594 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 5494538240 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 5368709120 IDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚4289 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4289 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚262 D;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚5264 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚59395584 I8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 H@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚665 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚223 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 5368709120 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚4293 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚2562113 NADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚58387428 IBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚60 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚11266666 E6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚322 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚429 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 5368709120 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 VCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚83303410068170 NBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚9574705 `ODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 778223907328 K>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59053056 R?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚83792236912128 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚59395584 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 122905600 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654602240 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚20643840 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚21498715 ODeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚262 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521206 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚262 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 trainer/global_step‚3049 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚10227756 J>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚9574646 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚7012590 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚209 TCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 735643442688 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚21498715 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 4517085184 WFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 734108810982 % _timestamp‚1.7866594667079637e+09 S@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚83792295965184 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚11266666 WDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚73204523959808 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚11266664 YFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚83056593469440 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚10227756 G@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚59 B6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚322 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 [JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 735685390848 NGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚57 SFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚60484608 G@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚60 D=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚60 WFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 735643442688 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚322 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚10232049 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚4293 NADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚59053056 ]JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚83056610574336 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 831258624 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 YFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚82569242869760 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 845178880 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚41348068 _runtime‚ 815.23824943 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚21494420 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚10232049 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17104896 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 5368709120 D5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5264 I@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚4876 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚7012592 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 ]JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚82569259909120â^FŠ®&‚ ¨M J>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚9574646 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚7012590 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚209 TCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 735643442688 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚21498715 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 4517085184 WFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 734108810982 % _timestamp‚1.7866594667079637e+09 S@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚83792295965184 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚11266666 WDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚73204523959808 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚11266664 YFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚83056593469440 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚10227756 G@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚59 B6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚322 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 [JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 735685390848 NGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚57 SFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚60484608 G@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚60 D=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚60 WFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 735643442688 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚322 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚10232049 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚4293 NADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚59053056 ]JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚83056610574336 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 831258624 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 YFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚82569242869760 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 845178880 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚41348068 _runtime‚ 815.23824943 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚21494420 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚10232049 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17104896 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 5368709120 D5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5264 I@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚4876 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚7012592 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 ]JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚82569259909120 ZGDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚83056610574336 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚41948160 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 H@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚665 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚223 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 5368709120 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚4293 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚2562113 NADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚58387428 IBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚60 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚11266666 E6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚322 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚429 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 5368709120 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 VCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚83303410068170 NBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚9574705 `ODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 778223907328 K>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59053056 R?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚83792236912128 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚59395584 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 122905600 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654602240 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚20643840 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚21498715 ODeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚262 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521206 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚262 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 trainer/global_step‚3049 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚10227756ùŒ“H0ƒ + _wandbruntime‚815  _runtime‚815ŽH˜p„ ê epoch‚7 % _timestamp‚1.7866594667089357e+09 _step‚250 0loss/FRt1/assignment_loss‚0.9169775036749478 %loss/total_loss‚2.816205248332713 trainer/global_step‚3049 _runtime‚ 815.238741741 0loss/FRt2/assignment_loss‚0.8138062473266463 /loss/FRt1/detection_loss‚0.5717490145539754 /loss/FRt2/detection_loss‚0.5136724827771438T¸ ªu… ï epoch‚7 % _timestamp‚1.7866594667089357e+09 _step‚250 0loss/FRt1/assignment_loss‚0.9169775036749478 %loss/total_loss‚2.816205248332713 trainer/global_step‚3049 _runtime‚ 815.238741741 0loss/FRt2/assignment_loss‚0.8138062473266463 /loss/FRt1/detection_loss‚0.5717490145539754 /loss/FRt2/detection_loss‚0.5136724827771438úrÄ3 † ‚(:‹ ”…ùÓûì“á gpu.0.gpu‚63.0 gpu.0.memory‚11, gpu.0.memoryAllocated‚7.0313262939453125) gpu.0.memoryAllocatedBytes‚ 6039863296 gpu.0.temp‚34.0 gpu.0.powerWatts‚119.973' gpu.0.enforcedPowerLimitWatts‚500.0 gpu.0.powerPercentÛ]Ü›‰‚23.9946 gpu.0.smClock‚1410 gpu.0.memoryClock‚1593$ gpu.0.correctedMemoryErrors‚1955# gpu.0.uncorrectedMemoryErrors‚0 7àˆ‡ ‚(:ý ”…ùÓ­Øçã% disk.nvme0n1.out‚7609.03857421875 disk.dm-0.in‚ 237.05078125 proc.memory.rssMB‚ 2315.28125* disk./.usagePercent‚29.972154009299178 disk.sda3.out‚ 17.23046875 disk.dm-1.in‚ 7126.828125* proc.memory.percent‚14.131355285644531 cpu‚26.762090507509537" disk.nvme0c0n1.in‚ 237.05078125' disk.nvme0c0n1.out‚7609.03857421875 disk.sda1.in‚0 disk.drbd2165.out‚0" disk.dm-0.out‚7609.03857421875$ memory_percent‚65.06805419921875# proc.memory.availableMB‚5723.25 network.recv‚ 1.722227e+06$ disk./.usageGB‚446.6067543029785 disk.sda.out‚ 17.23046875 disk.sda2.in‚0 disk.nvme0n1.in‚ 237.05078125 disk.dm-1.out‚ 17.23046875 disk.drbd2165.in‚0 proc.cpu.threads‚17 network.sent‚ 4.112123e+06 disk.sda3.in‚ 7126.828125 disk.sda.in‚ 7126.828125 disk.sda1.out‚0 disk.sda2.out‚0׿ 0ˆ + _wandbruntime‚827  _runtime‚827sc"Ɖ À & lr-AdamW/pg1‚0.0014309059253393855 & lr-AdamW/pg2‚0.0014309059253393855 trainer/global_step‚3099 % _timestamp‚1.7866594789070716e+09 _runtime‚ 827.437092161 _step‚251Kd5'ËŠ Å & lr-AdamW/pg1‚0.0014309059253393855 & lr-AdamW/pg2‚0.0014309059253393855 trainer/global_step‚3099 % _timestamp‚1.7866594789070716e+09 _runtime‚ 827.437092161 _step‚251ûx¢‡«0‹ + _wandbruntime‚827  _runtime‚827ך 'Œ šO PDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚9722829 L@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚9722768 ]LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 745481396998 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 5494538240 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 5368709120 _runtime‚ 827.441903944 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚59395584 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚429 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 4552982528 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 5368709120 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚262 IBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚60 F?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚60 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚11440761 PCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚59049984 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚10393858 ^MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 790200691200 _step‚252 M@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚59049984 Q>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚85093387161088 D;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚5264 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚2602192 XEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚85093446211072 UHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚64681984 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚322 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚322 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚10389571 [HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚83852346753024 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 4517085184 KDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚60 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 122905600 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚10389571 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17104896 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 831258624 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 125829120 IDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4876 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 4628521206 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚20643840 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 PCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚58387380 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 125829120 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚262 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 4654602240 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚4287 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚209 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 5494538240 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 % _timestamp‚1.7866594789080138e+09 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 ]JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚74345926533632 PIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚59 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚2602133 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 845178880 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 BDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚11440763 TADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚84597786802002 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 IDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4289 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚41348020 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚4287 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚223 YHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 745440048978 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 122905600 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚11440763 YFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚74345861851648 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚44038144 XEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚84597845189382 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 4654602240 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 4552982528 C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚4289 ]LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 747041084928 YHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 746999139840 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚21834621 IBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚61 `MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚73555661160448 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚7120635 TADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚85093387161088 KBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚4876 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 122306526 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 125829120 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚322 trainer/global_step‚3099 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚10393858 JBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚665 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚7120637 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 _LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚83852363792384 bQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 790244729344 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 5368709120 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚41945088 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚41945088ÓGÍ¥' ŸO PIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚59 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚2602133 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 845178880 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 BDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚11440763 TADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚84597786802002 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 IDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4289 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚41348020 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚4287 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚223 YHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 745440048978 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 122905600 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚11440763 YFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚74345861851648 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚44038144 XEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚84597845189382 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 4654602240 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 4552982528 C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚4289 ]LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 747041084928 YHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 746999139840 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚21834621 IBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚61 `MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚73555661160448 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚7120635 TADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚85093387161088 KBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚4876 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 122306526 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 125829120 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚322 trainer/global_step‚3099 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚10393858 JBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚665 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚7120637 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 _LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚83852363792384 bQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 790244729344 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 5368709120 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚41945088 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚41945088 PDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚9722829 L@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚9722768 ]LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 745481396998 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 5494538240 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 5368709120 _runtime‚ 827.441903944 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚59395584 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚429 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 4552982528 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 5368709120 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚262 IBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚60 F?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚60 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚11440761 PCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚59049984 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚10393858 ^MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 790200691200 _step‚252 M@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚59049984 Q>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚85093387161088 D;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚5264 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚2602192 XEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚85093446211072 UHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚64681984 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚322 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚322 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚10389571 [HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚83852346753024 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 4517085184 KDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚60 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 122905600 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚10389571 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17104896 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 831258624 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 125829120 IDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4876 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 4628521206 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚20643840 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 PCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚58387380 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 125829120 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚262 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 4654602240 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚4287 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚209 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 5494538240 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 % _timestamp‚1.7866594789080138e+09 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 ]JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚74345926533632üÈ‚k¼0Ž + _wandbruntime‚827  _runtime‚827µo$©& £M SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 122306526 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 125829120 IBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚60 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚10397151 ODeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521206 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 5494538240 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 _step‚253 NBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚9725828 ^KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚73578708533248 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 5368709120 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚322 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚11444316 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚4293 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚223 SFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚66776064 _runtime‚ 827.599492831 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 125829120 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 R?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚85119940567552 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654602240 ]JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚84372727463936 `ODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 790491330048 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 [JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 747272156672 WDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚74369153731072 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 S@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚85119999620608 YFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚84372710359040 WFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 747230208512 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚262 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 4654602240 VCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚84372710359040 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 B9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚5264 I@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚4876 bODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚73578729177088 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17104896 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚41948160 E5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5264 TCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 747230208512 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚21841467 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚10397151 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 5368709120 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚4295 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 [JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 745711969658 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚4293 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚429 NGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚65 B6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚322 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚262 G@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚60 trainer/global_step‚3099 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 NADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚58387428 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 4517085184 $ _timestamp‚1.786659479069173e+09 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚21841467 K>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59053056 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17104896 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚2602921 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚209 \KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 790445197824 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 ]JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚83878549585920 G@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚67 XGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 747272156672 NADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚59053056 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 122905600 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚21837172 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚322 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚21837172 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 125829120 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚11444314 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚10392858 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 4552982528 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚41948160 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚10392858 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚7122842 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚46132224 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 5368709120 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 122905600 EDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚9725761 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚262 DDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚11444316 VCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚85119999620608SåŒ÷“ ¨M QEDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚7122840 >5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5264 TCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 747230208512 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚21841467 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚10397151 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 5368709120 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚4295 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 [JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 745711969658 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚4293 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚429 NGDeviceStatsMonitor.on_train_batch_end/inactive_split.sma]‘­]#ll_pool.current‚65 B6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚322 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚262 G@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚60 trainer/global_step‚3099 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 NADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚58387428 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 4517085184 $ _timestamp‚1.786659479069173e+09 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚21841467 K>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59053056 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17104896 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚2602921 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚209 \KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 790445197824 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 ]JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚83878549585920 G@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚67 XGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 747272156672 NADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚59053056 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 122905600 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚21837172 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚322 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚21837172 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 125829120 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚11444314 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚10392858 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 4552982528 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚41948160 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚10392858 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚7122842 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚46132224 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 5368709120 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 122905600 EDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚9725761 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚262 DDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚11444316 VCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚85119999620608 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 122306526 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 125829120 IBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚60 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚10397151 ODeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521206 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 5494538240 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 _step‚253 NBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚9725828 ^KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚73578708533248 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 5368709120 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚322 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚11444316 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚4293 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚223 SFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚66776064 _runtime‚ 827.599492831 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 125829120 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 R?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚85119940567552 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654602240 ]JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚84372727463936 `ODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 790491330048 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 [JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 747272156672 WDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚74369153731072 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 S@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚85119999620608 YFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚84372710359040 WFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 747230208512 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚262 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 4654602240 VCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚84372710359040 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 B9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚5264 I@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚4876 bODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚73578729177088 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17104896 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚41948160 EDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4876 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚2642768 ^MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 802476903424 TADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚85918638289410 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 125829120 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚41945088 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚7230887 dQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚74708050444288 PCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚58387380 XEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚86421149866496 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚11618411 _runtime‚ 839.811140704 _step‚256 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚223 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚2642701 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 XEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚85918696676790 ]LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 757043207606 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 831258624 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 IDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚86421090816512 PIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚67 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚4287 YHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 758585905664 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚11618413 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 YHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 757001859586 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚262 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 4552982528 D;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚5264 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚11618411 ZIDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 758627850752 E8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚22173084 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 122905600 PDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚9873655 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚262 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 122905600 YFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚75510506703872 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚322 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚41945088 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚322 XEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚85662504910848 _LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚85662522015744 KBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚4876 L@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚9873586 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚429 C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚4289 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17104896 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 5368709120 UBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚86421149866496 @7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚5264 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 4654602240 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚10558960 BDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 4552982528 TADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚86421090816512 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚59395584 ]JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚75510573483008 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 5494538240 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 5494538240 \IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚85662522015744 _LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚85161653469184 VEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 758585905664 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4289 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 4517085184 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 5494538240 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 5368709120 F?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚60 trainer/global_step‚3149 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚22177373 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 125829120 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 125829120 M@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚59049984 ]LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 758627850752 IDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4876 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚2642768 ^MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 802476903424 TADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚85918638289410 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 125829120 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚41945088 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚7230887 dQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚74708050444288 PCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚58387380 XEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚86421149866496 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚11618411 _runtime‚ 839.811140704 _step‚256 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚223 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚2642701 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 XEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚85918696676790 ]LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 757043207606 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 831258624 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 IDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚86421090816512 PIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚67 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚4287 YHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 758585905664 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚11618413 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 YHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 757001859586 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚262 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 4552982528 D;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚5264 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚11618411 ZIDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 758627850752 E8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚22173084 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 122905600 PDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚9873655 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚262 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 122905600 YFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚75510506703872 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚322 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚41945088 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚322 XEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚85662504910848 _LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚85662522015744 KBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚4876 L@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚9873586 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚429 C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚4289 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17104896 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 5368709120 UBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚86421149866496 @7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚5264 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 4654602240 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚10558960 BDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 4552982528 TADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚86421090816512 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚59395584 ]JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚75510573483008 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 5494538240 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 5494538240 \IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚85662522015744 _LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚85161653469184 VEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 758585905664 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4289 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 4517085184 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 5494538240 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 5368709120 F?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚60 trainer/global_step‚3149 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚22177373 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 125829120 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 125829120 M@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚59049984 ]LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 758627850752 I6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚322 VCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚85945113042986 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚11621964 NADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚59053056 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654602240 SFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚66776064 B8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 4654602240 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 4517085184 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 831258624 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 YFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚85187822223360 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚322 NADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚58387428 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚11621966 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚41948160 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚4293 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 % _timestamp‚1.7866594914478235e+09 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚4293 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚10557960 XGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 758858922496 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 4552982528 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚322 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 5494538240 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 DDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚9876595 NGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚67 \KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 802721934336 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 125829120 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 125829120 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚4295 >5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5264 ZGDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚85688844353536 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 5494538240 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 ODeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521206 S@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚86447703276032 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚41948160 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 trainer/global_step‚3149 [HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚75533865883648 WFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 757232432198 IBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚60 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚22184219 G@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚69 TCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 758816974336 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚22184219 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 125829120 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚10562253 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚22179924 B9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚5264 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚11621964 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚10557960 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 845178880 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 ]JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚85187839262720 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 _runtime‚ 839.978125658 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 5494538240 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 5368709120 EDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59053056^>k‹¯&œ ©M NADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚58387428 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚11621966 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚41948160 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚4293 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 % _timestamp‚1.7866594914478235e+09 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚4293 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚10557960 XGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 758858922496 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 4552982528 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚322 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 5494538240 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 DDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚9876595 NGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚67 \KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 802721934336 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 125829120 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 125829120 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚4295 >5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5264 ZGDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚85688844353536 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 5494538240 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 ODeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521206 S@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚86447703276032 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚41948160 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 trainer/global_step‚3149 [HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚75533865883648 WFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 757232432198 IBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚60 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚22184219 G@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚69 TCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 758816974336 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚22184219 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 125829120 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚10562253 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚22179924 B9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚5264 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚11621964 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚10557960 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 845178880 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 ]JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚85187839262720 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 _runtime‚ 839.978125658 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 5494538240 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 5368709120 EDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59053056 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 4552982528 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17104896 [JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 758858922496 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 122905600 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚10562253 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 VCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚86447703276032 R?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚86447644222976 YFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚85688827248640 ^KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚74731077173248 G@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚60 H@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚665 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚2643572 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚46132224 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚262 bODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚74731097817088 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚322 VCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚85945113042986 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚11621964 NADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚59053056 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654602240 SFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚66776064 B8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 4654602240 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 4517085184 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 831258624 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 YFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚85187822223360 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚322ä|)0 + _wandbruntime‚839  _runtime‚839€Õy pž ê 0loss/FRt1/assignment_loss‚1.0455486681359603 /loss/FRt1/detection_loss‚0.6074640160374136 .loss/FRt2/detection_loss‚0.501226774425359 epoch‚7 _runtime‚ 839.978792548 _step‚258 0loss/FRt2/assignment_loss‚0.8864930391362857 &loss/total_loss‚3.0407324977350187 trainer/global_step‚3149 % _timestamp‚1.7866594914489179e+09áz)ÕuŸ ï epoch‚7 _runtime‚ 839.978792548 _step‚258 0loss/FRt2/assignment_loss‚0.8864930391362857 &loss/total_loss‚3.0407324977350187 trainer/global_step‚3149 % _timestamp‚1.7866594914489179e+09 0loss/FRt1/assignment_loss‚1.0455486681359603 /loss/FRt1/detection_loss‚0.6074640160374136 .loss/FRt2/detection_loss‚0.501226774425359‚~ål¼¡  ‚(:– £…ùÓ‹œ¦á gpu.0.gpu‚57.0 gpu.0.memory‚11, gpu.0.memoryAllocated‚7.0313262939453125) gpu.0.memoryAllocatedBytes‚ 6039863296 gpu.0.temp‚37.0 gpu.0.powerWatts‚222.511' gpu.0.enforcedPowerLimitWatts‚500.0) gpu.0.powerPercent‚44.502199999999995 gpu.0.smClock‚1410 gpu.0.memoryClock‚1593$ gpu.0.correctedMemoryErrors‚1955# gpu.0.uncorrectedMemoryErrors‚0FÈâw‹¡ ‚(:€ £…ùÓÌ¡ã" disk.dm-0.out‚7873.09326171875$ memory_percent‚65.06712436676025 cpu‚26.768774771045354 network.recv‚ 1.755211e+06 disk.sda3.in‚ 8281.28515625 disk.sda.out‚ 17.63671875 disk.sda1.in‚0 disk.drbd2165.in‚0) proc.memory.availableMB‚ 5723.40234375 proc.memory.rssMB‚ 2315.28125* disk./.usagePercent‚29.972136088771776 disk.sda1.out‚0 disk.dm-1.out‚ 17.63671875 disk.drbd2165.out‚0* proc.memory.percent‚14.131355285644531 proc.cpu.threads‚17 network.sent‚ 4.187053e+06$ disk./.usageGB‚446.6064872741699 disk.sda3.out‚ 17.63671875' disk.nvme0c0n1.out‚7873.09326171875 disk.sda.in‚ 8281.28515625 disk.sda2.in‚0 disk.sda2.out‚0 disk.nvme0n1.in‚ 240.59375 disk.nvme0c0n1.in‚ 240.59375% disk.nvme0n1.out‚7873.09326171875 disk.dm-1.in‚ 8281.28515625 disk.dm-0.in‚ 240.593752UŸ=0¢ + _wandbruntime‚853  _runtime‚853“‡zÉÆ£ À & lr-AdamW/pg2‚0.0014257788011482237 trainer/global_step‚3199 % _timestamp‚1.7866595050357292e+09 _runtime‚ 853.565770976 _step‚259 & lr-AdamW/pg1‚0.0014257788011482237 šŠÌˤ Å trainer/global_step‚3199 % _timestamp‚1.7866595050357292e+09 _runtime‚ 853.565770976 _step‚259 & lr-AdamW/pg1‚0.0014257788011482237 & lr-AdamW/pg2‚0.0014257788011482237ƒHcø0¥ + _wandbruntime‚853  _runtime‚853œ 1ý¡'¦ ›O YHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 770172671488 ]JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚76675215488512 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 IDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚7341135 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚322 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚322 _step‚260 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚11796061 F>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚41945088 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 $ _timestamp‚1.786659505036653e+09 Q>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚87748794471936 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚2683908 BDeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4289 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 ]LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 768605018214 \IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚86978638905344 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 4517085184 [HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚86978621800448 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚22520125 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚11796063 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 5494538240 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚322 _runtime‚ 853.569889598 G>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4876 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚11796061 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚7341137 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚48232448 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 5494538240 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 5368709120 XEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚87748853521920 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚429 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚41348020 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 KDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚60 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚11796063 _LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚86978638905344 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 125829120 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚20643840 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚322 C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚4289 UBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚87748853521920 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚10724062 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚2683841 YHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 768563670194 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚223 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 5494538240 @7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚5264 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 4552982528 PCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚59049984 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 122905600 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 KBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚4876 PCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚58387380€0¡“§  O [HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚86978621800448 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚22520125 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚11796063 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 5494538240 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚322 _runtime‚ 853.569889598 G>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4876 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚11796061 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚7341137 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚48232448 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 5494538240 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 5368709120 XEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚87748853521920 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚429 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚41348020 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 KDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚60 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚11796063 _LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚86978638905344 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 125829120 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚20643840 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚322 C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚4289 UBDeviceStatsMonitor.onW¹~»_train_batch_start/active_bytes.all.allocated‚87748853521920 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚10724062 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚2683841 YHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 768563670194 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚223 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 5494538240 @7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚5264 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 4552982528 PCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚59049984 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 122905600 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 KBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚4876 PCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚58387380 YHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 770172671488 ]JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚76675215488512 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 IDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚7341135 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚322 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚322 _step‚260 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚11796061 F>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚41945088 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 $ _timestamp‚1.786659505036653e+09 Q>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚87748794471936 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚2683908 BDeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4289 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 ]LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 768605018214 \IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚86978638905344 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 4517085184„RG0¨ + _wandbruntime‚853  _runtime‚853†ý{ý«&© ¥M E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚429 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚322 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚11799616 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 4654602240 NADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚59053056 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 5494538240 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 D8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 VCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚87775406931456 [JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 770445688320 I@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚4876 G@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚62 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚322 _step‚261 VCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚87004944138240 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚11799614 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚10027997 WFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 768794242806 trainer/global_step‚3199 E5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5264 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 H@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚665 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚22522676 bODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚75883466457088 NADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚58387428 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17104896 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚4293 \KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 814992514560 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 125829120 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 [JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 768835590874 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚10723062 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚11799616 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚2684657 G@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚60 EDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654602240 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚322 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521206 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 4552982528 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚223 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 5368709120 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 122306526 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚4293 BDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚262 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 5494538240 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 4517085184 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 5368709120 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 125829120 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 845178880 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚20643840 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 125829120 _runtime‚ 853.725410396 R?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚87775347878400 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 ]JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚86497128939520 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚41348068 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 VCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚87265964530394 R?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚87265906142966 K>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59053056 O6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚322 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚41948160 TCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 770403740160 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚7343340 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚262 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚7343342 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚209 [HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚76698503006720 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚22526971 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚22522676 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 5494538240 $ _timestamp‚1.786659505195171e+09 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚4295 XGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 770445688320 B9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚5264 ^KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚75883445813248 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 D=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚60 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚10727355éø•k°&ª ªM REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚10727355 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚429 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚322 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚11799616 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 4654602240 NADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚59053056 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 5494538240 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 D8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 VCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚87775406931456 [JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 770445688320 I@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚4876 G@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚62 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚322 _step‚261 VCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚87004944138240 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚11799614 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚10027997 WFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 768794242806 trainer/global_step‚3199 E5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5264 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 H@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚665 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚22522676 bODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚75883466457088 NADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚58387428 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17104896 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚4293 \KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 814992514560 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 125829120 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 [JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 768835590874 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚10723062 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚11799616 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚2684657 G@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚60 EDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654602240 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚322 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521206 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 4552982528 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚223 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 5368709120 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 122306526 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚4293 BDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚262 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 5494538240 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 4517085184 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 5368709120 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 125829120 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 845178880 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚20643840 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 125829120 _runtime‚ 853.725410396 R?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚87775347878400 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 ]JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚86497128939520 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚41348068 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 VCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚87265964530394 R?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚87265906142966 K>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59053056 O6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚322 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚41948160 TCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 770403740160 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚7343340 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚262 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚7343342 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚209 [HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚76698503006720 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚22526971 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚22522676 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 5494538240 $ _timestamp‚1.786659505195171e+09 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚4295 XGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 770445688320 B9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚5264 ^KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚75883445813248 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 D=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚60…ý Ì©0« + _wandbruntime‚853  _runtime‚853ç)Œrn¬ è 0loss/FRt2/assignment_loss‚0.9385568720510188 epoch‚7 % _timestamp‚1.7866595051961396e+09 _runtime‚ 853.72594192 _step‚262 0loss/FRt1/assignment_loss‚0.9605769953738899 .loss/FRt1/detection_loss‚0.579210240743457 /loss/FRt2/detection_loss‚0.5552373295232315 %loss/total_loss‚3.033581437691597 trainer/global_step‚3199Ä-r…s­ í 0loss/FRt2/assignment_loss‚0.9385568720510188 epoch‚7 % _timestamp‚1.7866595051961396e+09 _runtime‚ 853.72594192 _step‚262 0loss/FRt1/assignment_loss‚0.9605769953738899 .loss/FRt1/detection_loss‚0.579210240743457 /loss/FRt2/detection_loss‚0.5552373295232315 %loss/total_loss‚3.033581437691597 trainer/global_step‚3199†ƒ•×¬ ® ‚(:• ²…ùÓãš÷à gpu.0.gpu‚56.0 gpu.0.memory‚6, gpu.0.memoryAllocated‚7.0313262939453125) gpu.0.memoryAllocatedBytes‚ 6039863296 gpu.0.temp‚35.0 gpu.0.powerWatts‚132.573' gpu.0.enforcedPowerLimitWatts‚500.0) gpu.0.powerPercent‚26.514599999999998 gpu.0.smClock‚1410 gpu.0.memoryClock‚1593$ gpu.0.correctedMemoryErrors‚1955# gpu.0.uncorrectedMemoryErrors‚0¸ û²¯ ‚(:ô ²…ùÓ°¯Øâ* proc.memory.percent‚14.321351051330566 proc.cpu.threads‚17) disk./.usagePercent‚29.97227868496835 disk.drbd2165.out‚0 disk.sda.in‚ 8284.8984375 disk.sda2.in‚0$ memory_percent‚65.25921821594238# proc.memory.rssMB‚ 2346.41015625 cpu‚26.779969229015055 network.sent‚ 4.28147e+06 disk.dm-0.out‚8050.669921875 disk.sda3.out‚ 18.60546875% disk.nvme0c0n1.out‚8050.669921875 disk.sda.out‚ 18.60546875 disk.nvme0n1.in‚264.6875# disk.nvme0n1.out‚8050.669921875 network.recv‚ 1.786647e+06 disk.dm-1.out‚ 18.60546875 disk.drbd2165.in‚0 disk.dm-0.in‚264.6875 disk.sda3.in‚ 8284.8984375 disk.sda2.out‚0$ disk./.usageGB‚446.6086120605469 disk.dm-1.in‚ 8284.8984375 disk.nvme0c0n1.in‚264.6875 disk.sda1.in‚0 disk.sda1.out‚0( proc.memory.availableMB‚ 5691.9296875I¨mó0° + _wandbruntime‚858  _runtime‚858õþÅmÚ± Ô 7validation_average_jet_accuracy‚0.27566149830818176 (loss/val_total_loss‚2.78709047979083 0Purity/2FRt/event_purity‚0.33438000082969666 *jet/accuracy_1_of_2‚0.5196694135665894 +particle/specificity‚0.6369928121566772 .Purity/*FRt/event_purity‚0.401568740606308 3Purity/0FRt/event_proportion‚0.3301943838596344 +particle/sensitivity‚0.5964279770851135 /particle/accuracy_1_of_2‚0.8859137296676636 Purity/0FRt/event_purity‚1 /particle/accuracy_2_of_2‚0.6425892114639282 epoch‚7 2Purity/2FRt/event_proportion‚0.040062315762043 +jet/accuracy_1_of_1‚0.27117645740509033 (particle/accuracy‚0.6226050853729248 _runtime‚ 858.614824647 2Purity/*FRt/event_proportion‚0.669805645942688 3Purity/1FRt/event_proportion‚0.6297432780265808 +validation_accuracy‚0.17268458008766174 _step‚263 ,Purity/*FRt/FRt_purity‚0.414579302072525 /Purity/1FRt/event_purity‚0.4057345986366272 -Purity/1FRt/FRt_purity‚0.4057345986366272 +jet/accuracy_2_of_2‚0.17268458008766174 trainer/global_step‚3199 % _timestamp‚1.7866595100846682e+09 .Purity/2FRt/FRt_purity‚0.48297709226608276 particle/accuracy_1_of_1‚1 'particle/f_score‚0.5286734700202942˜Ö¨ß² Ù +jet/accuracy_2_of_2‚0.17268458008766174 trainer/global_step‚3199 % _timestamp‚1.7866595100846682e+09 .Purity/2FRt/FRt_purity‚0.48297709226608276 particle/accuracy_1_of_1‚1 'particle/f_score‚0.5286734700202942 7validation_average_jet_accuracy‚0.27566149830818176 (loss/val_total_loss‚2.78709047979083 0Purity/2FRt/event_purity‚0.33438000082969666 *jet/accuracy_1_of_2‚0.5196694135665894 +particle/specificity‚0.6369928121566772 .Purity/*FRt/event_purity‚0.401568740606308 3Purity/0FRt/event_proportion‚0.3301943838596344 +particle/sensitivity‚0.5964279770851135 /particle/accuracy_1_of_2‚0.8859137296676636 Purity/0FRt/event_purity‚1 /particle/accuracy_2_of_2‚0.6425892114639282 epoch‚7 2Purity/2FRt/event_proportion‚0.040062315762043 +jet/accuracy_1_of_1‚0.27117645740509033 (particle/accuracy‚0.6226050853729248 _runtime‚ 858.614824647 2Purity/*FRt/event_proportion‚0.669805645942688 3Purity/1FRt/event_proportion‚0.6297432780265808 +validation_accuracy‚0.17268458008766174 _step‚263 ,Purity/*FRt/FRt_purity‚0.414579302072525 /Purity/1FRt/event_purity‚0.4057345986366272 -Purity/1FRt/FRt_purity‚0.4057345986366272‡"¡î³ ‚(:– Á…ùÓ†à£á gpu.0.gpu‚58.0 gpu.0.memory‚10, gpu.0.memoryAllocated‚7.0313262939453125) gpu.0.memoryAllocatedBytes‚ 6039863296 gpu.0.temp‚34.0 gpu.0.powerWatts‚120.402' gpu.0.enforcedPowerLimitWattâ}í;³s‚500.0) gpu.0.powerPercent‚24.080399999999997 gpu.0.smClock‚1410 gpu.0.memoryClock‚1593$ gpu.0.correctedMemoryErrors‚1955# gpu.0.uncorrectedMemoryErrors‚0ÑGH‚´ ‚(:÷ Á…ùÓȸ¬ä disk.drbd2165.in‚0 disk.drbd2165.out‚0 disk.sda3.in‚ 8284.8984375 disk.sda3.out‚ 18.79296875 cpu‚26.80362592586366 disk.dm-1.out‚ 18.79296875& disk.nvme0c0n1.out‚8274.6806640625 disk.sda.in‚ 8284.8984375 disk.sda.out‚ 18.79296875 disk.sda2.in‚0$ disk.nvme0n1.out‚8274.6806640625$ memory_percent‚65.67931175231934 network.sent‚ 4.350411e+06 network.recv‚ 1.813188e+06$ disk./.usageGB‚446.6079902648926! disk.dm-0.out‚8274.6806640625 disk.sda1.out‚0 disk.nvme0n1.in‚264.6875 proc.cpu.threads‚17) disk./.usagePercent‚29.97223695574027 disk.dm-0.in‚264.6875 disk.nvme0c0n1.in‚264.6875 disk.sda1.in‚0 disk.sda2.out‚0( proc.memory.availableMB‚ 5623.1015625# proc.memory.rssMB‚ 2353.84765625* proc.memory.percent‚14.366745948791504 disk.dm-1.in‚ 8284.8984375Ž¹æ»0µ + _wandbruntime‚871  _runtime‚871=1uåƶ À & lr-AdamW/pg1‚0.0014231500500122317 & lr-AdamW/pg2‚0.0014231500500122317 trainer/global_step‚3249 % _timestamp‚1.7866595230426288e+09 _runtime‚ 871.572630433 _step‚264LDz Ë· Å & lr-AdamW/pg1‚0.0014231500500122317 & lr-AdamW/pg2‚0.0014231500500122317 trainer/global_step‚3249 % _timestamp‚1.7866595230426288e+09 _runtime‚ 871.572630433 _step‚264ˆ”Î Q0¸ + _wandbruntime‚871  _runtime‚871x«Æ¢'¹ œO KDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚60 _step‚265 dQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚77312058032128 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 5368709120 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚262 XEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚89471562118656 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚22945675 `MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚77312037388288 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚322 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 5494538240 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 5494538240 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 122905600 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 845178880 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 ]LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 784232800928 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚41348020 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚262 ZIDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 785869108736 ]JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚78143207202304 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4289 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚41945088 @7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚5264 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 G>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4876 PCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚59049984 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 831258624 % _timestamp‚1.7866595230436127e+09 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 4654602240 D;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚5264 UHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚70973440 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚12034319 XEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚88685675905024 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚223 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 4517085184 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 5368709120 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚322 IDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 XEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚88945365115552 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 5494538240 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 4654602240 PCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚58387380 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚12034317 Q>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚89471503068672 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚2733961 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 122306526 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17104896 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚41945088 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17104896 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚4287 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚7492001 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚429 KBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚4876 UBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚89471562118656 [HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚88685675905024 YHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 784191452908 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 5368709120 YFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚78143136228864 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 125829120 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 4552982528 YHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 785827163648 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚7491999 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚20643840 [HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚88161115275264 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚262 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 E8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚22941386 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚4287 IDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚12034319 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚2734036 ^MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 831098840576 JBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚665 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 TADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚88945306728172 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 trainer/global_step‚3249 ]LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 785869108736 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚10911356 VEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 785827163648 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚10907069 M@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚59049984 \IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚88685693009920 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚10226037½ß„§'º ¡O TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚12034319 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚2734036 ^MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 831098840576 JBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚665 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 TADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚88945306728172 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 trainer/global_step‚3249 ]LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 785869108736 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚10911356 VEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 785827163648 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚10907069 M@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚59049984 \IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚88685693009920 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚10226037 KDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚60 _step‚265 dQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚77312058032128 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 5368709120 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚262 XEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚89471562118656 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚22945675 `MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚77312037388288 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚322 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 5494538240 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 5494538240 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 122905600 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 845178880 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 ]LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 784232800928 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚41348020 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚262 ZIDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 785869108736 ]JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚78143207202304 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4289 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚41945088 @7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚5264 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 G>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4876 PCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚59049984 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 831258624 % _timestamp‚1.7866595230436127e+09 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 4654602240 D;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚5264 UHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚70973440 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚12034319 XEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚88685675905024 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚223 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 4517085184 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 5368709120 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚322 IDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 XEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚88945365115552 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 5494538240 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 4654602240 PCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚58387380 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚12034317 Q>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚89471503068672 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚2733961 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 122306526 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17104896 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚41945088 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17104896 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚4287 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚7492001 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚429 KBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚4876 UBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚89471562118656 [HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚88685675905024 YHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 784191452908 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 5368709120 YFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚78143136228864 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 125829120 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 4552982528 YHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 785827163648 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚7491999 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚20643840 [HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚88161115275264 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚262 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 E8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚22941386 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚4287 IDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665‰;Ó«ê0» + _wandbruntime‚871  _runtime‚871[\ª5¬&¼ ¦M O5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5264 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚12037870 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17104896 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚10228941 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 5494538240 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 5368709120 K>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59053056 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚20643840 [JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 784463373588 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 4654602240 XGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 786100180480 ]JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚88712015347712 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚2734813 _runtime‚ 871.742627589 WFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 786058232320 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 ]JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚88187318108160 trainer/global_step‚3249 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚10910356 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚429 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚22948226 VCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚89498115528192 TCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 786058232320 YFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚88711998242816 I@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚4876 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 E6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚322 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚262 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚22952521 NADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚58387428 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚262 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚10914649 [JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 786100180480 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚223 BDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 \KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 831342102016 R?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚88971723094320 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 122306526 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚262 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚50326528 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚22948226 SFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚70970368 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚322 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚12037872 H@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚665 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 G@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚60 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 R?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚89498056475136 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚22952521 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚4293 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚2734737 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚322 bODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚77335105404928 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 831258624 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 EDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654602240 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17104896 B9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚5264 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521206 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 % _timestamp‚1.7866595232123086e+09 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚12037870 G@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚78 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0—4×Áh½ «M H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚4293 S@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚89498115528192 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚41948160 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚10914649 NGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚76 ^KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚77335084761088 YFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚88187301068800 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 VCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚88971781481748 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 OBDeviceStatsMonitor.on_train_bat²˜ÿâI#ch_end/inactive_split.all.allocated‚10229019 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚41348068 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654602240 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17104896 B9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚5264 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521206 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 % _timestamp‚1.7866595232123086e+09 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚12037870 G@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚78 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 O5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5264 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚12037870 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17104896 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚10228941 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 5494538240 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 5368709120 K>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59053056 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚20643840 [JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 784463373588 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 4654602240 XGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 786100180480 ]JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚88712015347712 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚2734813 _runtime‚ 871.742627589 WFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 786058232320 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 ]JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚88187318108160 trainer/global_step‚3249 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚10910356 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚429 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚22948226 VCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚89498115528192 TCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 786058232320 YFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚88711998242816 I@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚4876 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 E6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚322 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚262 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚22952521 NADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚58387428 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚262 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚10914649 [JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 786100180480 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚223 BDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 \KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 831342102016 R?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚88971723094320 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 122306526 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚262 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚50326528 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚22948226 SFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚70970368 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚322 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚12037872 H@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚665 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 G@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚60 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 R?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚89498056475136 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚22952521 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚4293 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚2734737 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚322 bODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚77335105404928 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 831258624 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 EDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4876 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚2775074 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚59395584 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 4552982528 ]LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 797455874560 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚10377323 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 122905600 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 _LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚90001809899520 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 5368709120 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚322 ZIDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 797455874560 XEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚90799265774080 JBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚665 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 IDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 125829120 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚322 % _timestamp‚1.7866595358211784e+09 TADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚90799206724096 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 BDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚90799206724096 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚10377392 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚7602251 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚20643840 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚262 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 F>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 XEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚90266216602960 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 5494538240 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚429 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚7602249 `MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚78464406028288 YHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 795753263516 PCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚59049984 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 _LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚89470421991424 [HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚90001792794624 PCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚58387380 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 125829120 E8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚23284138 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚12211967 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 831258624 ]LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 795794611536 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 4654602240 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 125829120 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17104896 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4289 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚209 ]JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚79307853019136 YFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚79307779948544 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚52426752 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 5368709120 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚11076458 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚322 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚41945088 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚4287 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 4517085184 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚262 F?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚60 _runtime‚ 884.351406587 IDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 125829120 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚322 % _timestamp‚1.7866595358211784e+09 TADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚90799206724096 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 BDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚90799206724096 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚10377392 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚7602251 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚20643840 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚262 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 F>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 XEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚90266216602960 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 5494538240 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚429 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚7602249 `MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚78464406028288 YHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 795753263516 PCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚59049984 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 _LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚89470421991424 [HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚90001792794624 PCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚58387380 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 125829120 E8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚23284138 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚12211967 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 831258624 ]LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 795794611536 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 4654602240 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 125829120 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17104896 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4289 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚209 ]JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚79307853019136 YFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚79307779948544 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚52426752 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 5368709120 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚11076458 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚322 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚41945088 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚4287 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 4517085184 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚262 F?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚60 _runtime‚ 884.351406587 IDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4876 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚2775074 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚59395584 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 4552982528 ]LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 797455874560 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚10377323 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 122905600 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 _LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚90001809899520 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 5368709120 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚322 ZIDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 797455874560 XEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚90799265774080 JBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚665 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 IDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚10380318 D=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚60 >5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5264 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 % _timestamp‚1.7866595359810822e+09 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚23295273 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚23290978 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚209 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚20643840 NADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚58387428 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 125829120 G@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚60 TCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 797644998144 H@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚665 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚11075458 NGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚68 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 125829120 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚262 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 4552982528 S@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚90825819183616 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚4293 J=DeviceStatsMonitor.on_train_batch_end/activ“è¼âÄe.large_pool.freed‚12215520 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 4552982528 WFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 797644998144 SFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚68873216 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚48229376 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚11075458 ODeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 5368709120 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚322 WDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚79331070615552 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 IBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚60 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚12215522 [JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 797686946304 YFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚89496590745600 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚4293 K>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59053056 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 ]JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚89496607784960 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 4517085184 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 5368709120 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚11079751 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚41948160 EDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚23295273 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚322 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654602240 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚322 trainer/global_step‚3299 _runtime‚ 884.511318625 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 831258624 B6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚322 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521206 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 125829120 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚223 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚12215520 \KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 843617214464 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 [JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 796025184196 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17104896 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚12215522 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚41948160 G@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚70 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 5494538240 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 5368709120ËvGø±&É «M 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚11075458 ODeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 5368709120 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚322 WDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚79331070615552 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 IBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚60 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚12215522 [JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 797686946304 YFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚89496590745600 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚4293 K>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59053056 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 ]JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚89496607784960 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 4517085184 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 5368709120 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚11079751 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚41948160 EDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚23295273 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚322 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654602240 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚322 trainer/global_step‚3299 _runtime‚ 884.511318625 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 831258624 B6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚322 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521206 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 125829120 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚223 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚12215520 \KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 843617214464 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 [JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 796025184196 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17104896 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚12215522 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚41948160 G@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚70 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 5494538240 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 5368709120 R?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚90825760130560 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚10380318 D=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚60 >5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5264 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 % _timestamp‚1.7866595359810822e+09 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚23295273 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚23290978 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚209 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚20643840 NADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚58387428 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 125829120 G@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚60 TCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 797644998144 H@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚665 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚11075458 NGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚68 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 125829120 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚262 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 4552982528 S@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚90825819183616 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚4293 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚12215520 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 4552982528 WFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 797644998144 SFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚68873216 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚48229376Žî’ÍÑ0Ê + _wandbruntime‚884  _runtime‚884lp bpË ê 0loss/FRt1/assignment_loss‚0.7523596112190692 0loss/FRt2/assignment_loss‚0.7436991798952955 /loss/FRt1/detection_loss‚0.5540365222846024 %loss/total_loss‚2.566741872534385 epoch‚8 % _timestamp‚1.7866595359821963e+09 /loss/FRt2/detection_loss‚0.5166465591354179 trainer/global_step‚3299 _runtime‚ 884.512042524 _step‚271³Úë“uÌ ï /loss/FRt1/detection_loss‚0.5540365222846024 %loss/total_loss‚2.566741872534385 epoch‚8 % _timestamp‚1.7866595359821963e+09 /loss/FRt2/detection_loss‚0.5166465591354179 trainer/global_step‚3299 _runtime‚ 884.512042524 _step‚271 0loss/FRt1/assignment_loss‚0.7523596112190692 0loss/FRt2/assignment_loss‚0.7436991798952955ó8ð¡Í ‚(:– Ð…ùÓíúˆá gpu.0.gpu‚65.0 gpu.0.memory‚12, gpu.0.memoryAllocated‚7.0313262939453125) gpu.0.memoryAllocatedBytes‚ 6039863296 gpu.0.temp‚35.0 gpu.0.powerWatts‚119.834' gpu.0.enforcedPowerLimitWatts‚500.0) gpu.0.powerPercent‚23.966800000000003 gpu.0.smClock‚1410 gpu.0.memoryClock‚1593$ gpu.0.correctedMemoryErrors‚1955# gpu.0.uncorrectedMemoryErrors‚0„ä öŽÎ ‚(:ƒ Ð…ùÓ̶§ã) disk./.usagePercent‚30.24175375155489 disk.dm-1.in‚ 8284.8984375 disk.dm-1.out‚ 19.25390625 disk.drbd2165.out‚0 disk.dm-0.in‚ 266.8671875 disk.sda.in‚ 8284.8984375$ disk.nvme0n1.out‚8613.2373046875) proc.memory.availableMB‚ 5622.51953125 disk.drbd2165.in‚0! disk.dm-0.out‚8613.2373046875 disk.sda3.out‚ 19.25390625! disk.nvme0c0n1.in‚ 266.8671875 disk.sda1.in‚0 disk.sda2.out‚0# proc.memory.rssMB‚ 2353.84765625* proc.memory.percent‚14.366745948791504 network.sent‚ 4.405925e+06% disk./.usageGB‚450.62398529052734 disk.nvme0n1.in‚ 266.8671875$ memory_percent‚65.68286418914795 network.recv‚ 1.845378e+06 disk.sda3.in‚ 8284.8984375& disk.nvme0c0n1.out‚8613.2373046875 disk.sda.out‚ 19.25390625 disk.sda1.out‚0 disk.sda2.in‚0 cpu‚26.808509785282034 proc.cpu.threads‚17Jo€0Ï + _wandbruntime‚897  _runtime‚897UYÆÐ À & lr-AdamW/pg1‚0.0014177630138474495 & lr-AdamW/pg2‚0.0014177630138474495 trainer/global_step‚3349 % _timestamp‚1.7866595484817939e+09 _runtime‚ 897.013889309 _step‚2726Ñ|ËÑ Å & lr-AdamW/pg1‚0.0014177630138474495 & lr-AdamW/pg2‚0.0014177630138474495 trainer/global_step‚3349 % _timestamp‚1.7866595484817939e+09 _runtime‚ 897.013889309 _step‚272E±áK0Ò + _wandbruntime‚897  _runtime‚897FÑŒð¡'Ó ›O TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚12389619 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚322 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚262 F?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚60 PCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚58387380 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 5494538240 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚41945088 KDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚60 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚7712499 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚2816413 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚41348020 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 [HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚91317909684224 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚223 dQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚79616795312128 [HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚90779694628864 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 4654602240 XEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚92126969429504 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚11241560 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚10528847 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17104896 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4289 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 845178880 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 5368709120 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 122905600 IBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚67 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚20643840 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 PCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚59049984 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 4552982528 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 XEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚91587068090368 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚322 ZIDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 809042640384 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚429 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚209 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 5368709120 IDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚92126910379520 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 4552982528 TADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚92126910379520 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 4517085184 UBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚92126969429504 UHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚62584832 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 122306526 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚11237273 YFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚80472429674496 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚12389619 F>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚59395584 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 125829120 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚2816348 ]JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚80472492259328 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 5368709120 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 125829120 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 122905600 YHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 809000695296 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚23631179 _LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚90779711668224 YHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 807315074124 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 5494538240 `MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚79616774668288 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 125829120 $ _timestamp‚1.786659548482496e+09 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚4287 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 4654602240 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 5494538240 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚262 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚41940992 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 4628521206 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚322 G>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4876 VEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 809000695296 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚12389617 IBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚60 E8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚23626890 XEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚91317909684224 _LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚91317926789120 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚41945088 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚10528914 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 831258624 _runtime‚ 897.013979159 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17104896 KBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚4876 PIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚65 bQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 855696947200 ^MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 855655006208 \IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚91317926789120 IDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚322 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚262 F?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚60 PCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚58387380 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 5494538240 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚41945088 KDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚60 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.larjV2žLge_pool.freed‚7712499 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚2816413 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚41348020 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 [HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚91317909684224 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚223 dQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚79616795312128 [HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚90779694628864 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 4654602240 XEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚92126969429504 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚11241560 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚10528847 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17104896 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4289 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 845178880 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 5368709120 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 122905600 IBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚67 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚20643840 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 PCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚59049984 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 4552982528 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 XEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚91587068090368 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚322 ZIDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 809042640384 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚429 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚209 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 5368709120 IDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚92126910379520 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 4552982528 TADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚92126910379520 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 4517085184 UBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚92126969429504 UHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚62584832 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 122306526 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚11237273 YFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚80472429674496 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚12389619 F>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚59395584 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 125829120 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚2816348 ]JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚80472492259328 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 5368709120 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 125829120 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 122905600 YHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 809000695296 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚23631179 _LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚90779711668224 YHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 807315074124 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 5494538240 `MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚79616774668288 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 125829120 $ _timestamp‚1.786659548482496e+09 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚4287 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 4654602240 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 5494538240 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚262 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚41940992 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 4628521206 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚322 G>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4876 VEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 809000695296 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚12389617 IBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚60 E8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚23626890 XEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚91317909684224 _LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚91317926789120 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0‘?‡ÿŽ0Õ + _wandbruntime‚897  _runtime‚897\®G„¬&Ö ¦M REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17104896 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 D=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚60 [HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚80495786036224 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 BDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚23638025 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 5368709120 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚262 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 125829120 DDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 122905600 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚59395584 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚322 trainer/global_step‚3349 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚41948160 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚11244853 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 5368709120 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚4295 ODeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654602240 YFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚91344232022016 I@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚4876 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚209 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 831258624 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 >5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5264 WFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 809231763968 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚7714704 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 5368709120 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚4293 XGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 809273712128 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚48229376 E6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚322 ]JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚90805897461760 S@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚92153522839040 TCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 809231763968 B9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚5264 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 845178880 % _timestamp‚1.7866595486494308e+09 ^KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚79639822041088 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚11244853 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚10531946 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 122306526 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚322 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚223 WFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 807545646736 bODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚79639842684928 YFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚90805880422400 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚23638025 ZGDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚91344249126912 `ODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 855943351296 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 [JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 807586994804 IBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚60 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 4654602240 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521206 VCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚91344232022016 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚11240560 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 125829120 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚4293 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 4517085184 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚322 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚262 K>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59053056 NADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚59053056 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 5494538240 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 4552982528 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚10531871 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 5494538240 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 NGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚73 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚2817167 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚20643840 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 5494538240 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚11240560 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 R?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚91613426069136 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚262 ]JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚91344249126912 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 _step‚274 H@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚665 _runtime‚ 897.179834602 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚236337301=ô±&× «M H@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚665 _runtime‚ 897.179834602 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚23633730 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17104896 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 D=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚60 [HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚80495786036224 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 BDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚23638025 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 5368709120 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚262 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 125829120 DDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 122905600 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚59395584 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚322 trainer/global_step‚3349 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚41948160 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚11244853 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 5368709120 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚4295 ODeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654602240 YFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚91344232022016 I@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚4876 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚209 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 831258624 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 >5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5264 WFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 809231763968 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚7714704 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 5368709120 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚4293 XGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 809273712128 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚48229376 E6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚322 ]JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚90805897461760 S@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚92153522839040 TCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 809231763968 B9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚5264 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 845178880 % _timestamp‚1.7866595486494308e+09 ^KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚79639822041088 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚11244853 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚10531946 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 122306526 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚322 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚223 WFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 807545646736 bODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚79639842684928 YFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚90805880422400 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚23638025 ZGDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚91344249126912 `ODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 855943351296 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 [JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 807586994804 IBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚60 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 4654602240 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521206 VCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚91344232022016 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚11240560 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 125829120 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚4293 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 4517085184 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚322 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚262 K>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59053056 NADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚59053056 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 5494538240 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 4552982528 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚10531871 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 5494538240 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 NGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚73 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚2817167 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚20643840 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 5494538240 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚11240560 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 R?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚91613426069136 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚262 ]JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚91344249126912 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 _step‚274’%ð9d0Ø + _wandbruntime‚897  _runtime‚897ÔôàŠnÙ è 0loss/FRt2/assignment_loss‚0.8179720266082742 %loss/total_loss‚2.733484989401795 trainer/global_step‚3349 _runtime‚ 897.18223048 0loss/FRt1/assignment_loss‚0.8720059525516038 /loss/FRt1/detection_loss‚0.5662250161972747 /loss/FRt2/detection_loss‚0.4772819940446425 epoch‚8 $ _timestamp‚1.786659548650453e+09 _step‚275Ô²«sÚ í trainer/global_step‚3349 _runtime‚ 897.18223048 0loss/FRt1/assignment_loss‚0.8720059525516038 /loss/FRt1/detection_loss‚0.5662250161972747 /loss/FRt2/detection_loss‚0.4772819940446425 epoch‚8 $ _timestamp‚1.786659548650453e+09 _step‚275 0loss/FRt2/assignment_loss‚0.8179720266082742 %loss/total_loss‚2.733484989401795“l¿P Û ‚(:• ß…ùÓé¤á gpu.0.gpu‚50.0 gpu.0.memory‚9, gpu.0.memoryAllocated‚7.0313262939453125) gpu.0.memoryAllocatedBytes‚ 6039863296 gpu.0.temp‚37.0 gpu.0.powerWatts‚212.771' gpu.0.enforcedPowerLimitWatts‚500.0) gpu.0.powerPercent‚42.554199999999994 gpu.0.smClock‚1410 gpu.0.memoryClock‚1593$ gpu.0.correctedMemoryErrors‚1955# gpu.0.uncorrectedMemoryErrors‚0Ín#Ü ‚(:‚ ß…ùÓ‰²¤ã) proc.memory.availableMB‚ 5622.95703125# proc.memory.rssMB‚ 2353.84765625 cpu‚26.812910566065256 disk.sda.in‚ 8284.8984375 disk.sda2.in‚0 disk.dm-1.out‚ 19.43359375* proc.memory.percent‚14.366745948791504 proc.cpu.threads‚17& disk.nvme0c0n1.out‚8620.5712890625 disk.sda.out‚ 19.43359375 network.recv‚ 1.87822e+06* disk./.usagePercent‚30.258606215518856 disk.drbd2165.out‚0 disk.dm-0.in‚ 266.8671875! disk.dm-0.out‚8620.5712890625 disk.sda3.in‚ 8284.8984375 network.sent‚ 4.501262e+06 disk.drbd2165.in‚0 disk.sda3.out‚ 19.43359375 disk.sda1.in‚0 disk.sda2.out‚0 disk.nvme0n1.in‚ 266.8671875$ disk.nvme0n1.out‚8620.5712890625 disk.dm-1.in‚ 8284.8984375! disk.nvme0c0n1.in‚ 266.8671875$ disk./.usageGB‚450.8750991821289 disk.sda1.out‚0$ memory_percent‚65.68019390106201n{•l0Ý + _wandbruntime‚910  _runtime‚910g]g£ÆÞ À trainer/global_step‚3399 % _timestamp‚1.7866595615409555e+09 _runtime‚ 910.072870733 _step‚276 & lr-AdamW/pg1‚0.0014150050748179253 & lr-AdamW/pg2‚0.0014150050748179253mÖm…Ëß Å & lr-AdamW/pg1‚0.0014150050748179253 & lr-AdamW/pg2‚0.0014150050748179253 trainer/global_step‚3399 % _timestamp‚1.7866595615409555e+09 _runtime‚ 910.072870733 _step‚276”K÷=å0à + _wandbruntime‚910  _runtime‚910¶ý–á œO L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚12567267 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚2857629 YFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚81637072208384 `MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚80769143308288 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚12567269 F>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 M@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚59049984 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚10680380 TADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚92907861190396 _step‚277 ]LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 820629406208 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚12567267 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 PCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚58387380 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚262 ]JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚81637145278976 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 [HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚92088984305664 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 4517085184 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚223 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 122905600 PIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚70 UHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚73070592 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚20643840 _LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚92089001m#&T 345024 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚7822749 ^MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 867928900096 Q>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚93454614034944 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 4552982528 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 125829120 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 125829120 IBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚72 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚11402375 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 122905600 BDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 4628521206 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚429 C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚4289 YHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 820587461120 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 \IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚92634043678720 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17104896 JBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚665 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚7822751 dQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚80769163952128 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 5368709120 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4289 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚10680308 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚322 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚41945088 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 bQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 867981326848 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚52426752 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚262 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 % _timestamp‚1.7866595615415943e+09 XEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚93454673084928 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚11406662 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚4287 ]LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 818918232752 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 5494538240 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 125829120 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚12567269 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚4287 _LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚92634043678720 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 831258624 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚23973931 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 F?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚60 _runtime‚ 910.072967526 KBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚4876 E8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚23969642 ZIDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 820629406208 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚2857559 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚322 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 TADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚93454614034944 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 122306526 G>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4876 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 YHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 818876884732 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 5494538240 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 XEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚92907919577776 KDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚60 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 4552982528 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚209 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚322©øK§'â ¡O ]LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 820629406208 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚12567267 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 PCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚58387380 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚262 ]JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚81637145278976 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 [HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚92088984305664 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 4517085184 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚223 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 122905600 PIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚70 UHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚73070592 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚20643840 _LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚92089001345024 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚7822749 ^MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 867928900096 Q>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚93454614034944 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 4552982528 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 125829120 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 125829120 IBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚72 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚11402375 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 122905600 BDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 4628521206 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚429 C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚4289 YHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 820587461120 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 \IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚92634043678720 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17104896 JBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚665 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚7822751 dQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚80769163952128 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 5368709120 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4289 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚10680308 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚322 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚41945088 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 bQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 867981326848 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚52426752 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚262 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 % _timestamp‚1.7866595615415943e+09 XEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚93454673084928 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚11406662 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚4287 ]LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 818918232752 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 5494538240 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 125829120 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚12567269 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚4287 _LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚92634043678720 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 831258624 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚23973931 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 F?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚60 _runtime‚ 910.072967526 KBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚4876 E8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚23969642 ZIDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 820629406208 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚2857559 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚322 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 TADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚93454614034944 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 122306526 G>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4876 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 YHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 818876884732 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 5494538240 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 XEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚92907919577776 KDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚60 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 4552982528 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚209 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚322 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚12567267 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚2857629 YFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚81637072208384 `MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚80769143308288 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚12567269 F>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 M@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚59049984 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚10680380 TADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚92907861190396 _step‚277•äê¶^0ã + _wandbruntime‚910  _runtime‚910$€ 7¬&ä ¦M >5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5264 NADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚59053056 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚11405662 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 122905600 SFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚64678912 % _timestamp‚1.7866595617026098e+09 VCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚92934335943972 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚11409955 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚10683409 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚10683344 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚23980777 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚11409955 WDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚81660368118272 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚11405662 B9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚5264 ^KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚80792190681088 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521206 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 5494538240 E8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 5368709120 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 125829120 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚322 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚262 D=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚60 EDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚23980777 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 TCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 820818529792 G@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚65 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚2858390 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 4517085184 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 G@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚60 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚23976482 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚322 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 125829120 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚12570822 I@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚4876 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654602240 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 122905600 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 831258624 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚322 ODeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59053056 S@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚93481226494464 _step‚278 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚41948160 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚41948160 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚44035072 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 YFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚92660348911616 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚23976482 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 [HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚81660432797184 ]JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚92115187138560 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚4295 ZGDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚92660366016512 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚2858453 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚20643840 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 trainer/global_step‚3399 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 4552982528DZNå «M ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 TCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 820818529792 G@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚65 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚2858390 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 4517085184 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 G@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚60 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚23976482 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚322 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 125829120 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚12570822 I@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚4876 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654602240 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 122905600 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 831258624 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚322 ODeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59053056 S@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚93481226494464 _step‚278 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚41948160 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚41948160 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚44035072 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 YFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚92660348911616 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚23976482 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 [HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚81660432797184 ]JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚92115187138560 A8DeviceStatsMonitor.on_train_batch_end/active.all.curren5>q_ct‚4295 ZGDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚92660366016512 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚2858453 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚20643840 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 trainer/global_step‚3399 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 4552982528 >5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5264 NADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚59053056 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚11405662 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 122905600 SFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚64678912 % _timestamp‚1.7866595617026098e+09 VCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚92934335943972 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚11409955 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚10683409 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚10683344 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚23980777 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚11409955 WDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚81660368118272 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚11405662 B9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚5264 ^KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚80792190681088 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521206 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 5494538240 E8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 5368709120 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 125829120 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚322 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚262 D=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚60 EDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚23980777–TÊZI0æ + _wandbruntime‚910  _runtime‚910­Ìâ»oç é /loss/FRt1/detection_loss‚0.5823491290370213 /loss/FRt2/detection_loss‚0.5095513619793307 %loss/total_loss‚2.711805785193875 epoch‚8 _step‚279 0loss/FRt1/assignment_loss‚0.8771495781023085 0loss/FRt2/assignment_loss‚0.7427557160752143 trainer/global_step‚3399 $ _timestamp‚1.786659561703568e+09 _runtime‚ 910.233374603Q ¤tè î 0loss/FRt1/assignment_loss‚0.8771495781023085 0loss/FRt2/assignment_loss‚0.7427557160752143 trainer/global_step‚3399 $ _timestamp‚1.786659561703568e+09 _runtime‚ 910.233374603 /loss/FRt1/detection_loss‚0.5823491290370213 /loss/FRt2/detection_loss‚0.5095513619793307 %loss/total_loss‚2.711805785193875 epoch‚8 _step‚279—Ä嵐é ‚(:– î…ùÓ±ÿà gpu.0.gpu‚57.0 gpu.0.memory‚11, gpu.0.memoryAllocated‚7.0313262939453125) gpu.0.memoryAllocatedBytes‚ 6039863296 gpu.0.temp‚36.0 gpu.0.powerWatts‚227.091' gpu.0.enforcedPowerLimitWatts‚500.0) gpu.0.powerPercent‚45.418200000000006 gpu.0.smClock‚1410 gpu.0.memoryClock‚1593$ gpu.0.correctedMemoryErrors‚1955# gpu.0.uncorrectedMemoryErrors‚0´ÉžŒê ‚(: î…ùÓÑúöâ disk.sda3.out‚ 19.609375 disk.sda2.in‚0 disk.sda2.out‚0 disk.dm-1.out‚ 19.609375 disk.drbd2165.out‚0" disk.dm-0.out‚8990.59326171875$ memory_percent‚65.68219661712646# proc.memory.rssMB‚ 2353.84765625 disk.sda.in‚ 8284.8984375 disk.sda.out‚ 19.609375 disk.sda1.in‚0 disk.nvme0n1.in‚ 266.8671875% disk.nvme0n1.out‚8990.59326171875* proc.memory.percent‚14.366745948791504 cpu‚26.817979201430934 proc.cpu.threads‚17* disk./.usagePercent‚30.257476966285324% disk./.usageGB‚450.85827255249023! disk.nvme0c0n1.in‚ 266.8671875' disk.nvme0c0n1.out‚8990.59326171875 disk.sda1.out‚0 disk.drbd2165.in‚0 disk.dm-0.in‚ 266.8671875 network.sent‚ 4.576327e+06 network.recv‚ 1.911156e+06 disk.sda3.in‚ 8284.8984375 disk.dm-1.in‚ 8284.8984375) proc.memory.availableMB‚ 5622.62890625TtG0ë + _wandbruntime‚923  _runtime‚923mR‰TÅì ¿ & lr-AdamW/pg1‚0.0014122044237551085 & lr-AdamW/pg2‚0.0014122044237551085 trainer/global_step‚3449 $ _timestamp‚1.786659574690511e+09 _runtime‚ 923.220570899 _step‚280nnÊ Êí Ä & lr-AdamW/pg1‚0.0014122044237551085 & lr-AdamW/pg2‚0.0014122044237551085 trainer/global_step‚3449 $ _timestamp‚1.786659574690511e+09 _runtime‚ 923.220570899 _step‚280˜äT« 0î + _wandbruntime‚923  _runtime‚923ž`z¢'ï œO MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4289 IDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚41348020 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚262 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 4654602240 VEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 832174226944 [HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚93950143463424 ]LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 832216172032 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚12744917 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 _LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚93398291021824 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17104896 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚7932999 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚12744919 IBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚75 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚59395584 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 122306526 C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚4289 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚4287 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚2898603 dQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚81921532592128 IDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4876 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚12744919 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚10831677 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚2898676 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 XEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚94228771065184 YHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 830438695340 F?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚60 trainer/global_step‚3449 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚209 UHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚68876288 BDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚94782317690368 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚429 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚322 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚11567477 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚20643840 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 ]LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 830480043360 _step‚281 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚262 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚41945088 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚10831602 JBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚665 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 5494538240 [HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚93398273982464 @7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚5264 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 4552982528 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 122905600 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 M@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚59049984 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 122905600 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚24316683 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 PCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚58387380 UBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚94782376740352 ]JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚82801762794496 % _timestamp‚1.7866595746915286e+09 \IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚93950160568320 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚7933001 YFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚82801693918208 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 125829120 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚322' o§'ð ¡O \IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚93950160568320 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚7933001 YFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚82801693918208 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 125829120 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚322 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4289 IDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚41348020 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚262 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 4654602240 VEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 832174226944 [HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚93950143463424 ]LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 832216172032 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚12744917 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 _LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚93398291021824 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17104896 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚7932999 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚12744919 IBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚75 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚59395584 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 122306526 C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚4289 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚4287 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚2898603 dQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚81921532592128 IDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4876 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚12744919 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚10831677 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚2898676 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 XEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚94228771065184 YHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 830438695340 F?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚60 trainer/global_step‚3449 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚209 UHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚68876288 BDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚94782317690368 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚429 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚322 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚11567477 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚20643840 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 ]LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 830480043360 _step‚281 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚262 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚41945088 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚10831602 JBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚665 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 5494538240 [HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚93398273982464 @7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚5264 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 4552982528 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 122905600 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 M@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚59049984 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 122905600 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚24316683 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 PCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚58387380 UBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚94782376740352 ]JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚82801762794496 % _timestamp‚1.7866595746915286e+09™!œº0ñ + _wandbruntime‚923  _runtime‚923"(­“ò ¦M >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 5494538240 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚12748472 I@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚4876 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚41348068 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654602240 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚11575057 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚223 [HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚82825058668544 BDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521206 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 5368709120 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 4552982528 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚209 ^KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚81944559321088 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚24319234 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚12748470 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 122905600 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚24323529 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚54520832 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚41948160 bODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚81944579964928 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 5494538240 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚322 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚262 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚4293 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 ]JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚93424476815360 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 5494538240 VCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚93976465801216 WFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 830669267952 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 5368709120 G@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚60 EDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 122306526 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚12748470 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 trainer/global_step‚3449 WDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚82824983503872 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚59395584 _runtime‚ 923.394330533 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚4293 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚20643840 >5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5264 ODeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚10834601 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 125829120 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚4295 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚12748472 D=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚60 K>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59053056 `ODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 880478703616 \KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 880424182784 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚322 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚322 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0Y]†±&ó «M LDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚223 [HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚82825058668544 BDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521206 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 5368709120 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 4552982528 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚209 ^KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚81944559321088 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚24319234 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚12748470 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 122905600 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚24323529 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚54520832 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚41948160 bODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚81944579964928 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 5494538240 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚322 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚262 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚4293 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 ]JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚93424476815360 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 5494538240 VCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚93976465801216 WFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 830669267952 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 5368709120 G@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚60 EDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 122306526 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚12748470 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 trainer/global_step‚3449 WDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚82824983503872 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚59395584 _runtime‚ 923.394330533 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚4293 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚20643840 >5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5264 ODeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚10834601 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 125829120 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚4295 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚12748472 D=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚60 K>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59053056 `ODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 880478703616 \KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 880424182784 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚322 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚322 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 5494538240 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚12748472 I@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚4876 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚41348068 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654602240 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚11575057š‘¼V0ô + _wandbruntime‚923  _runtime‚923î×±Eoõ é %loss/total_loss‚2.769520545224537 trainer/global_step‚3449 $ _timestamp‚1.786659574864956e+09 _runtime‚ 923.394825679 0loss/FRt1/assignment_loss‚0.9047683673625082 /loss/FRt1/detection_loss‚0.5501145972639584 epoch‚8 _step‚283 0loss/FRt2/assignment_loss‚0.7973595191463917 /loss/FRt2/detection_loss‚0.5172780614516788¨` ótö î _runtime‚ 923.394825679 0loss/FRt1/assignment_loss‚0.9047683673625082 /loss/FRt1/detection_loss‚0.5501145972639584 epoch‚8 _step‚283 0loss/FRt2/assignment_loss‚0.7973595191463917 /loss/FRt2/detection_loss‚0.5172780614516788 %loss/total_loss‚2.769520545224537 trainer/global_step‚3449 $ _timestamp‚1.786659574864956e+09›f}ý–÷ ‚(:‹ ý…ùÓ冶ã gpu.0.gpu‚61.0 gpu.0.memory‚11, gpu.0.memoryAllocated‚7.0313262939453125) gpu.0.memoryAllocatedBytes‚ 6039863296 gpu.0.temp‚37.0 gpu.0.powerWatts‚226.299' gpu.0.enforcedPowerLimitWatts‚500.0 gpu.0.powerPercent‚45.2598 gpu.0.smClock‚1410 gpu.0.memoryClock‚1593$ gpu.0.correctedMemoryErrors‚1955# gpu.0.uncorrectedMemoryErrors‚0ü ïø ‚(:… ý…ùÓºêŽä' disk.nvme0c0n1.out‚9201.72119140625 disk.sda2.out‚0# proc.memory.rssMB‚ 2353.84765625 cpu‚26.821767596969266 network.recv‚ 1.944189e+06* disk./.usagePercent‚30.257464421916147 disk.dm-1.in‚ 8284.8984375 disk.dm-0.in‚ 266.8671875" disk.dm-0.out‚9201.72119140625 disk.sda.in‚ 8284.8984375 disk.sda1.in‚0* proc.memory.percent‚14.366745948791504$ disk./.usageGB‚450.8580856323242 disk.drbd2165.out‚0! disk.nvme0c0n1.in‚ 266.8671875 disk.sda1.out‚0 disk.sda2.in‚0$ memory_percent‚65.68245887756348( proc.memory.availableMB‚ 5622.5859375 proc.cpu.threads‚17 network.sent‚ 4.651527e+06 disk.drbd2165.in‚0 disk.sda3.in‚ 8284.8984375 disk.sda3.out‚ 19.82421875 disk.sda.out‚ 19.82421875 disk.nvme0n1.in‚ 266.8671875% disk.nvme0n1.out‚9201.72119140625 disk.dm-1.out‚ 19.82421875ªoSæ0ù + _wandbruntime‚935  _runtime‚935²£÷Äú ¾ & lr-AdamW/pg1‚0.0014093612405395842 & lr-AdamW/pg2‚0.0014093612405395842 trainer/global_step‚3499 # _timestamp‚1.78665958743771e+09 _runtime‚ 935.967926351 _step‚284q@ñÉû à & lr-AdamW/pg2‚0.0014093612405395842 trainer/global_step‚3499 # _timestamp‚1.78665958743771e+09 _runtime‚ 935.967926351 _step‚284 & lr-AdamW/pg1‚0.0014093612405395842œO¿ñ0ü + _wandbruntime‚935  _runtime‚935P:u›¢'ý œO @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚322 TADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚95549564165212 _step‚285 _LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚95266277457920 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 4517085184 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 125829120 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚322 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 IBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚60 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚10983324 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚8043249 _LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚94707580698624 YHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 843760992768 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 122905600 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚223 M@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚59049984 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17104896 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚12922567 UBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚96110080395776 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚59395584 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚11732579 VEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 843760992768 PCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚59049984 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚11732579 YFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚83966355465216 YHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 842000505948 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 122306526 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚11736866 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4289 XEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚95549622552592 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚24659435 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚2940018 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 F>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 4654602240 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 5368709120 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚322 KDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚60 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 5368709120 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚322 IDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4876 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 4654602240 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 4628521206 ZIDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 843802937856 IDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚96110021345792 XEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚95266260353024 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 122905600 [HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚95266260353024 ^MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 892474876928 IBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚57 ]JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚83966418050048 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0¦Læþ ¡O PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚12922569 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚262 TADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚96110021345792 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 845178880 F?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚60 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 5494538240 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 125829120 JBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚665 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚429 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚4287 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 5494538240 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚262 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚262 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚12922569 PCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚58387380 D;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚5264 Q>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚96110021345792 XEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚95266260353024 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 122905600 [HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚95266260353024 ^MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 892474876928 IBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚57 ]JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚83966418050048 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚322 TADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚95549564165212 _step‚285 _LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚95266277457920 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 4517085184 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 125829120 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚322 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 IBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚60 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚10983324 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚8043249 _LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚94707580698624 YHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 843760992768 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 122905600 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚223 M@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚59049984 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17104896 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚12922567 UBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚96110080395776 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚59395584 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚11732579 VEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 843760992768 PCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚59049984 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚11732579 YFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.?ª¦®”freed‚83966355465216 YHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 842000505948 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 122306526 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚11736866 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4289 XEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚95549622552592 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚24659435 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚2940018 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 F>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 4654602240 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 5368709120 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚322 KDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚60 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 5368709120 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚322 IDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4876 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 4654602240 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 4628521206 ZIDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 843802937856 I5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5264 EDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚10986259 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚262 G@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚60 _runtime‚ 936.163789714 ZGDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚95292599795712 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚10986330 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚59395584 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚11740159 I@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚4876 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 845178880 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17104896 [JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 844034009600 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 122905600 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚12926120 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚209 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 NADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚58387428 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 122306526 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚48229376 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 5494538240 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚24666281 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚24661986 WDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚83989642789888 ]JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚94733766492160 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚11735866 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚12926120 SFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚68873216 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚322 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚41348068 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 trainer/global_step‚3499 \KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 892714828800 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 4654602240 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚4293 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 5368709120 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚262 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚4293 E6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚322 K>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59053056 VCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚96136633805312 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚12926122 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521206 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚322 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚2940874 ODeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 S@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚96136633805312 ]JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚95292599795712 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚8045454 VCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚95576038918788 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 125829120 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 R?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚95575980531360 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 5368709120 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚262 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚8045456 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17104896 G@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚71 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 R?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚96136574752256 YFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚95292582690816 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 WFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 842231078560 D=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚60 $ _timestamp‚1.786659587633169e+09 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 H@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚665 ^KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚83096927961088 XGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 844034009600 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 4552982528 B9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚5264 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚20643840 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚24661986 TCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 843992061440 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚2940805 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 BDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654602240 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 5494538240 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚12926122 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚41948160 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚11735866 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚4295 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 _step‚286:DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚24666281 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚24661986 WDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚83989642789888 ]JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚94733766492160 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚11735866 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚12926120 SFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚68873216 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚322 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚41348068 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 trainer/global_step‚3499 \KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 892714828800 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 4654602240 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚4293 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 5368709120 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚262 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚4293 E6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚322 K>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59053056 VCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚96136633805312 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚12926122 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521206 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚322 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚2940874 ODeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 S@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚96136633805312 ]JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚95292599795712 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚8045454 VCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚95576038918788 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 125829120 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 R?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚95575980531360 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 5368709120 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚262 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚8045456 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17104896 G@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚71 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 R?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚96136574752256 YFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚95292582690816 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 WFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 842231078560 D=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚60 $ _timestamp‚1.786659587633169e+09 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 H@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚665 ^KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚83096927961088 XGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 844034009600 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 4552982528 B9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚5264 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚20643840 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚24661986 TCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 843992061440 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚2940805 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 BDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654602240 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 5494538240 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚12926122 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚41948160 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚11735866 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚4295 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 _step‚286 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚24666281 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚41948160 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 5368709120 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 125829120 >5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5264 EDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚10986259 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚262 G@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚60 _runtime‚ 936.163789714 ZGDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚95292599795712 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚10986330 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚59395584 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚11740159 I@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚4876 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 845178880 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17104896 [JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 844034009600 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 122905600 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚12926120 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚209 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 NADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚58387428 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 122306526žbý0‚ + _wandbruntime‚936  _runtime‚936ñV´]oƒ é trainer/global_step‚3499 % _timestamp‚1.7866595876342998e+09 _runtime‚ 936.164384038 _step‚287 .loss/FRt1/assignment_loss‚0.88089764866231 0loss/FRt2/assignment_loss‚0.7637469475227153 /loss/FRt2/detection_loss‚0.5028204319832437 &loss/total_loss‚2.6915160063546724 epoch‚8 /loss/FRt1/detection_loss‚0.5440509781864036`<ët„ î .loss/FRt1/assignment_loss‚0.88089764866231 0loss/FRt2/assignment_loss‚0.7637469475227153 /loss/FRt2/detection_loss‚0.5028204319832437 &loss/total_loss‚2.6915160063546724 epoch‚8 /loss/FRt1/detection_loss‚0.5440509781864036 trainer/global_step‚3499 % _timestamp‚1.7866595876342998e+09 _runtime‚ 936.164384038 _step‚287Ÿ¡gÀ … ‚(:• Œ†ùÓṌá gpu.0.gpu‚31.0 gpu.0.memory‚4, gpu.0.memoryAllocated‚7.0313262939453125) gpu.0.memoryAllocatedBytes‚ 6039863296 gpu.0.temp‚37.0 gpu.0.powerWatts‚121.968' gpu.0.enforcedPowerLimitWatts‚500.0) gpu.0.powerPercent‚24.393600000000003 gpu.0.smClock‚1410 gpu.0.memoryClock‚1593$ gpu.0.correctedMemoryErrors‚1955# gpu.0.uncorrectedMemoryErrors‚0/'N‚Ž† ‚(:ƒ Œ†ùÓî½ã' disk.nvme0c0n1.out‚9247.35400390625 disk.sda1.in‚0 disk.nvme0n1.in‚ 269.8203125% disk.nvme0n1.out‚9247.35400390625 disk.dm-1.out‚ 19.9765625 disk.sda3.in‚ 8288.76953125 disk.sda1.out‚0 disk.dm-1.in‚ 8288.76953125 disk.drbd2165.in‚0 disk.drbd2165.out‚0 disk.dm-0.in‚ 269.8203125" disk.dm-0.out‚9247.35400390625 cpu‚26.826563264464916 network.sent‚ 4.726246e+06 network.recv‚ 1.976969e+06 disk.sda.in‚ 8288.76953125 disk.sda2.in‚0# memory_percent‚65.6822681427002( proc.memory.availableMB‚ 5622.6171875# proc.memory.rssMB‚ 2353.84765625* proc.memory.percent‚14.366745948791504) disk./.usagePercent‚30.25745878975039$ disk./.usageGB‚450.8580017089844 disk.sda.out‚ 19.9765625 proc.cpu.threads‚17 disk.sda2.out‚0 disk.sda3.out‚ 19.9765625! disk.nvme0c0n1.in‚ 269.8203125*#70‡ + _wandbruntime‚948  _runtime‚948øéƈ À & lr-AdamW/pg2‚0.0014064757077836968 trainer/global_step‚3549 % _timestamp‚1.7866596003611293e+09 _runtime‚ 948.891178661 _step‚288 & lr-AdamW/pg1‚0.0014064757077836968$P÷4ˉ Å % _timestamp‚1.7866596003611293e+09 _runtime‚ 948.891178661 _step‚288 & lr-AdamW/pg1‚0.0014064757077836968 & lr-AdamW/pg2‚0.0014064757077836968 trainer/global_step‚3549  ]åÝ0Š + _wandbruntime‚948  _runtime‚948zú/ÁM‹ œO RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 122905600 YHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 855347758592 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 122905600 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚13100219 ]JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚85131048971264 ]LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 853603664576 _step‚289 XEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚96582377242624 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17104896 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚13100219 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4289 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 831258624 bQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 904779099136 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚322 JBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚665 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚11901968 IBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚60 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 4552982528 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 5368709120 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚262 IDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚97437725001216 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚13100217 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚59395584 XEDeviceStatsMonitor.on_train_biSíUatch_start/requested_bytes.all.allocated‚96870474040000 PCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚58387380 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 5494538240 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚13100217 \IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚96582394347520 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚41945088 KBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚4876 C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚4289 @7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚5264 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 4552982528 [HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚96582377242624 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚262 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚25002187 YFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚85130984289280 YHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 853562316556 XEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚97437784051200 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 4654602240 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚41945088 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 F>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 4628521206 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 122306526 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 125829120 PIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚70 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚2981238 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 5368709120 _runtime‚ 948.892404588 TADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚97437725001216 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚11897681 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚8153501 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 5494538240 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚11897681 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚4287 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚20643840 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 KDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚60 _LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚96016870375424 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚11134809 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚223 UHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚64681984 TADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚96870415652620 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 125829120 VEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 855347758592 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚11901968 M@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚59049984 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 845178880 ^MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 904735060992 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 G>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4876 UBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚97437784051200 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17104896 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 125829120 [HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚96016853336064 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚2981308 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚209 dQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚84226269872128 BDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚97437725001216 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚13100217 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚59395584 XEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚96870474040000 PCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚58387380 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 5494538240 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚13100217 \IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚96582394347520 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚41945088 KBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚4876 C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚4289 @7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚5264 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 4552982528 [HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚96582377242624 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚262 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚25002187 YFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚85130984289280 YHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 853562316556 XEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚97437784051200 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 4654602240 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚41945088 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 F>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 4628521206 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 122306526 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 125829120 PIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚70 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚2981238 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 5368709120 _runtime‚ 948.892404588 TADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚97437725001216 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚11897681 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚8153501 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 5494538240 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚11897681 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚4287 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚20643840 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 KDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚60 _LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚96016870375424 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚11134809 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚223 UHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚64681984 TADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚96870415652620 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 125829120 VEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 855347758592 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚11901968 M@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚59049984 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 845178880 ^MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 904735060992 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 G>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4876 UBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚97437784051200 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17104896 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 125829120 [HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚96016853336064 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚2981308 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚209 dQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚84226269872128 BDeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4289 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 831258624 bQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 904779099136 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚322 JBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚665 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚11901968 IBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚60 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 4552982528 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 5368709120 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚262 I8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 R?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚96896832018768 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 5494538240 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚8155704 NGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚60 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚2982024 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 4517085184 [JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 853834237236 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 O6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚322 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚223 K>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59053056 TCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 855578827264 YFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚96608699580416 ^KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚84249296601088 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 5494538240 ]JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚96608716685312 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17104896 [JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 855620775424 bODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚84249317244928 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚41948160 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚25009033 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚4293 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚11900968 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚20643840 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚262 >5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5264 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 122905600 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚13103772 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚429 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚4293 VCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚97464337460736 YFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚96043039129600 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚13103770 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 4654602240 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚41948160 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚262 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚11900968 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚11137728 WDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚85154275881984 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 5368709120 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 5368709120 _runtime‚ 949.059553581 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚13103772 XGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 855620775424 R?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚97464278407680 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚13103770 `ODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 905023315968 B6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚322 VCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚96608699580416 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 VCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚96896890406196 NADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚58387428 I@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚4876 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚2982084 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 845178880 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521206 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚262 WFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 855578827264 EDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚11905261 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚11905261 trainer/global_step‚3549 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 125829120 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚25009033 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 4552982528 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚209 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654602240 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 WFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 853792889168ÒvQ «M B6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚322 VCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚96608699580416 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 VCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚96896890406196 NADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚58387428 I@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚4876 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚2982084 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 845178880 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521206 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚262 WFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 855578827264 EDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚11905261 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚11905261 trainer/global_step‚3549 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 125829120 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚25009033 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 4552982528 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚209 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654602240 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 WFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 853792889168 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚4295 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚59395584 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚322 ZGDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚96608716685312 H@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚665 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 R?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚96896832018768 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 5494538240 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚8155704 NGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚60 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚2982024 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 4517085184 [JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 853834237236 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 O6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚322 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚223 K>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59053056 TCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 855578827264 YFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚96608699580416 ^KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚84249296601088 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 5494538240 ]JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚96608716685312 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17104896 [JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 855620775424 bODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚8424¬Ñ²‘¬ 9317244928 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚41948160 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚25009033 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚4293 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚11900968 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚20643840 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚262 >5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5264 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 122905600 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚13103772 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚429 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚4293 VCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚97464337460736 YFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚96043039129600 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚13103770 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 4654602240 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚41948160 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚262 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚11900968 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚11137728 WDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚85154275881984 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 5368709120 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 5368709120 _runtime‚ 949.059553581 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚13103772 XGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 855620775424 R?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚97464278407680 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚13103770 `ODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 905023315968¢¶ýs0 + _wandbruntime‚949  _runtime‚949ÎG¾·o‘ é $ _timestamp‚1.786659600530265e+09 0loss/FRt1/assignment_loss‚0.8534994970158258 0loss/FRt2/assignment_loss‚0.7593902800729148 epoch‚8 trainer/global_step‚3549 _runtime‚ 949.060061812 _step‚291 /loss/FRt1/detection_loss‚0.5297103756677268 /loss/FRt2/detection_loss‚0.5346613421402271 %loss/total_loss‚2.677261494896695×[”t’ î /loss/FRt2/detection_loss‚0.5346613421402271 %loss/total_loss‚2.677261494896695 $ _timestamp‚1.786659600530265e+09 0loss/FRt1/assignment_loss‚0.8534994970158258 0loss/FRt2/assignment_loss‚0.7593902800729148 epoch‚8 trainer/global_step‚3549 _runtime‚ 949.060061812 _step‚291 /loss/FRt1/detection_loss‚0.5297103756677268£4ðfŽ–“ ‚(:‹ ›†ùÓƒ»¤á gpu.0.gpu‚55.0 gpu.0.memory‚11, gpu.0.memoryAllocated‚7.0313262939453125) gpu.0.memoryAllocatedBytes‚ 6039863296 gpu.0.temp‚34.0 gpu.0.powerWatts‚120.113' gpu.0.enforcedPowerLimitWatts‚500.0 gpu.0.powerPercent‚24.0226 gpu.0.smClock‚1410 gpu.0.memoryClock‚1593$ gpu.0.correctedMemoryErrors‚1955# gpu.0.uncorrectedMemoryErrors‚00¦ùtŒ” ‚(: ›†ùÓ¶“ã disk.sda.in‚ 9337.50390625 disk.sda1.in‚0 disk.dm-1.in‚ 9336.765625 disk.dm-1.out‚ 20.1953125 disk.drbd2165.out‚0 network.recv‚ 1.99649e+06 disk.sda3.in‚ 9337.50390625 disk.sda2.in‚0 disk.drbd2165.in‚0 disk.dm-0.in‚ 284.37109375! disk.dm-0.out‚9507.9716796875# proc.memory.rssMB‚ 2354.22265625 proc.cpu.threads‚17 disk.sda3.out‚ 20.1953125 disk.sda.out‚ 20.1953125 disk.sda1.out‚0$ disk.nvme0n1.out‚9507.9716796875' proc.memory.availableMB‚ 5622.109375* proc.memory.percent‚14.369034767150879 cpu‚26.829186257491045 network.sent‚ 4.789331e+06" disk.nvme0c0n1.in‚ 284.37109375& disk.nvme0c0n1.out‚9507.9716796875 disk.sda2.out‚0 disk.nvme0n1.in‚ 284.37109375$ memory_percent‚65.68536758422852) disk./.usagePercent‚30.25750205502368% disk./.usageGB‚450.85864639282227¿¤“›0• + _wandbruntime‚961  _runtime‚961|YŸÆ– À trainer/global_step‚3599 % _timestamp‚1.7866596134253862e+09 _runtime‚ 961.955505843 _step‚292 & lr-AdamW/pg1‚0.0014035480108198235 & lr-AdamW/pg2‚0.0014035480108198235í?XˆË— Å & lr-AdamW/pg1‚0.0014035480108198235 & lr-AdamW/pg2‚0.0014035480108198235 trainer/global_step‚3599 % _timestamp‚1.7866596134253862e+09 _runtime‚ 961.955505843 _step‚292¤¥ÓUq0˜ + _wandbruntime‚961  _runtime‚961o_UŠ¡'™ ›O WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 125829120 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚262 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 YHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 866934524416 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚11285985 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚41348020 G>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4876 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚262 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 XEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚97898494132224 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚13277867 PIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚73 dQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚85378638512128 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚322 _step‚293 XEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚98765487706624 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚13277869 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚4287 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 4628521206 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚322 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 5368709120 @7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚5264 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚3022234 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚3022161 TADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚98191267140028 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 122306526 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚12062783 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 122905600 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚209 YFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚86295615428096 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 4654602240 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚25344939 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 _runtime‚ 961.959921365 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚223 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚4287 \IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚97898511237120 VEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 866934524416 ^MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 916997559808 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 Q>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚98765428656640 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 5494538240 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 125829120 $ _timestamp‚1.786659613426493e+09 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 4654602240 C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚4289 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 BDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 E8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚25340650 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17104896 IDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 ZIDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 866976469504 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚8263751 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚59395584 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17104896 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚322 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 5494538240 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚11285910 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚52426752 _LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚97326160052224 [HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚97898494132224 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 trainer/global_step‚3599 JBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚665 [HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚97326143012864 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 4517085184 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 4552982528 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚41945088 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚322 IDeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4289 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚20643840 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 5368709120 F?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚60 ]LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 866976469504 IBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚75 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 KDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚60 PCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚59049984 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 122905600 UHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚73070592 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 YHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 865124127164 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚41945088 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚429 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0¸§‘7¦'š  O GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 trainer/global_step‚3599 JBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚665 [HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚97326143012864 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 4517085184 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 4552982528 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚41945088 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚322 IDeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4289 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚20643840 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 5368709120 F?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚60 ]LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 866976469504 IBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚75 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 KDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚60 PCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚59049984 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 122905600 UHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚73070592 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 YHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 865124127164 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚41945088 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚429 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 125829120 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚262 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 YHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 866934524416 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚11285985 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚41348020 G>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4876 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚262 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 XEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚97898494132224 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚13277867 PIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚73 dQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚85378638512128 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚322 _step‚293 XEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚98765487706624 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚13277869 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚4287 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 4628521206 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚322 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 5368709120 @7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚5264 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚3022234 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚3022161 TADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚98191267140028 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 122306526 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚12062783 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 122905600 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚209 YFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚86295615428096 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 4654602240 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚25344939 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 _runtime‚ 961.959921365 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚223 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚4287 \IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚97898511237120 VEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 866934524416 ^MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 916997559808 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 Q>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚98765428656640 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 5494538240 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 125829120 $ _timestamp‚1.786659613426493e+09 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 4654602240 C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚4289 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 BDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 E8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚25340650 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17104896 IDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 ZIDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 866976469504 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚8263751 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚59395584 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17104896 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚322 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 5494538240 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚11285910 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚52426752 _LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚97326160052224 [HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚97898494132224¥ð0› + _wandbruntime‚962  _runtime‚962‰üŠ,Uœ ¦M J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 4654602240 [HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚86318978802176 NADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚58387428 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 % _timestamp‚1.7866596136121864e+09 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚4293 \KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 917246785024 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 ZGDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚97924833574912 bODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚85401685884928 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚322 B5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5264 DDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚25351785 G@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚60 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 E8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 IBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚60 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚322 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 ]JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚97924833574912 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚8265956 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 [JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 865396047844 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚262 D=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚60 `ODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 917292917248 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 5368709120 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚223 EDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521206 ]JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚97352345845760 TCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 867165593088 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17104896 H@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚665 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚209 G@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚60 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654602240 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚262 QBDeviceStatsMonitor.on_tr˜»ˆW ain_batch_end/active_bytes.large_pool.peak‚ 4552982528 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚4293 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 5494538240 S@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚98792041116160 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 5368709120 trainer/global_step‚3599 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 5368709120 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 125829120 K>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59053056 VCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚98792041116160 [JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 867207541248 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚25347490 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚25347490 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚12070363 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚11288970 WDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚86318912026112 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚322 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 122905600 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 122905600 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 125829120 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚13281420 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚11288910 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚59395584 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚41348068 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17104896 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚41948160 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚3022956 ^KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚85401665241088 _step‚294 SFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚66776064 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 4517085184[±& «M ]JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚97924833574912 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚8265956 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 [JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 865396047844 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚262 D=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚60 `ODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 917292917248 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 5368709120 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚223 EDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521206 ]JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚97352345845760 TCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 867165593088 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17104896 H@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚665 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚209 G@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚60 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654602240 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚262 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 4552982528 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚4293 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 5494538240 S@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚98792041116160 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 5368709120 trainer/global_step‚3599 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 5368709120 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 125829120 K>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59053056 VCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚98792041116160 [JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 867207541248 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚25347490 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚25347490 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚12070363 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚11288970 WDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚86318912026112 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚322 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 122905600 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 122905600 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 125829120 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚13281420 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚11288910 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚59395584 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚41348068 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17104896 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚41948160 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚3022956 ^KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚85401665241088 _step‚294 SFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚66776064 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 4517085184 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 4654602240 [HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚86318978802176 NADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚58387428 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 % _timestamp‚1.7866596136121864e+09 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚4293 \KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 917246785024 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 ZGDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚97924833574912 bODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚85401685884928 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚322 B5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5264 DDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚25351785 G@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚60 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 E8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 IBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚60 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚322 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0¦©9mç0ž + _wandbruntime‚962  _runtime‚962ÃahœqŸ ë _step‚295 /loss/FRt1/detection_loss‚0.5461201023988284 /loss/FRt2/detection_loss‚0.5248174760436718 &loss/total_loss‚2.6551846597966926 epoch‚8 0loss/FRt1/assignment_loss‚0.8368410505089092 0loss/FRt2/assignment_loss‚0.7474060308452835 trainer/global_step‚3599 % _timestamp‚1.7866596136134117e+09 _runtime‚ 962.143296593 2Ï$v  ð 0loss/FRt1/assignment_loss‚0.8368410505089092 0loss/FRt2/assignment_loss‚0.7474060308452835 trainer/global_step‚3599 % _timestamp‚1.7866596136134117e+09 _runtime‚ 962.143296593 _step‚295 /loss/FRt1/detection_loss‚0.5461201023988284 /loss/FRt2/detection_loss‚0.5248174760436718 &loss/total_loss‚2.6551846597966926 epoch‚8§sÚ×ò0¡ + _wandbruntime‚967  _runtime‚967TÜbØ¢ Ò )loss/val_total_loss‚2.735494762305054 /Purity/*FRt/event_purity‚0.4210452437400818 % _timestamp‚1.7866596189870422e+09 3Purity/0FRt/event_proportion‚0.3301943838596344 /Purity/2FRt/event_purity‚0.3626179099082947 2Purity/2FRt/event_proportion‚0.040062315762043 *jet/accuracy_1_of_2‚0.5420977473258972 /particle/accuracy_2_of_2‚0.5925465822219849 +validation_accuracy‚0.18036232888698578 _runtime‚ 967.517138636 /Purity/1FRt/event_purity‚0.4246657192707062 ,Purity/2FRt/FRt_purity‚0.509332001209259 /particle/accuracy_1_of_2‚0.8694079518318176 +particle/sensitivity‚0.5806770920753479 .Purity/*FRt/FRt_purity‚0.43431326746940613 particle/accuracy_1_of_1‚1 2Purity/*FRt/event_proportion‚0.669805645942688 Purity/0FRt/event_purity‚1 3Purity/1FRt/event_proportion‚0.6297432780265808 (particle/accuracy‚0.6381603479385376 6validation_average_jet_accuracy‚0.2889347970485687 &particle/f_score‚0.532490611076355 epoch‚8 trainer/global_step‚3599 _step‚296 -Purity/1FRt/FRt_purity‚0.4246657192707062 *jet/accuracy_1_of_1‚0.2843235433101654 +jet/accuracy_2_of_2‚0.18036232888698578 +particle/specificity‚0.6697884202003479ÝTOÝ£ × -Purity/1FRt/FRt_purity‚0.4246657192707062 *jet/accuracy_1_of_1‚0.2843235433101654 +jet/accuracy_2_of_2‚0.18036232888698578 +particle/specificity‚0.6697884202003479 )loss/val_total_loss‚2.735494762305054 /Purity/*FRt/event_purity‚0.4210452437400818 % _timestamp‚1.7866596189870422e+09 3Purity/0FRt/event_proportion‚0.3301943838596344 /Purity/2FRt/event_purity‚0.3626179099082947 2Purity/2FRt/event_proportion‚0.040062315762043 *jet/accuracy_1_of_2‚0.5420977473258972 /particle/accuracy_2_of_2‚0.5925465822219849 +validation_accuracy‚0.18036232888698578 _runtime‚ 967.517138636 /Purity/1FRt/event_purity‚0.4246657192707062 ,Purity/2FRt/FRt_purity‚0.509332001209259 /particle/accuracy_1_of_2‚0.8694079518318176 +particle/sensitivity‚0.5806770920753479 .Purity/*FRt/FRt_purity‚0.43431326746940613 particle/accuracy_1_of_1‚1 2Purity/*FRt/event_proportion‚0.669805645942688 Purity/0FRt/event_purity‚1 3Purity/1FRt/event_proportion‚0.6297432780265808 (particle/accuracy‚0.6381603479385376 6validation_average_jet_accuracy‚0.2889347970485687 &particle/f_score‚0.532490611076355 epoch‚8 trainer/global_step‚3599 _step‚296¨ 9¸J–¤ ‚(:‹ ª†ùÓ¶Öªá gpu.0.gpu‚64.0 gpu.0.memory‚12, gpu.0.memoryAllocated‚7.0313262939453125) gpu.0.memoryAllocatedBytes‚ 6039863296 gpu.0.temp‚36.0 gpu.0.powerWatts‚134.263' gpu.0.enforcedPowerLimitWatts‚500.0 gpu.0.powerPercent‚26.8526 gpu.0.smClock‚1410 gpu.0.memoryClock‚1593$ gpu.0.correctedMemoryErrors‚1955# gpu.0.uncorrectedMemoryErrors‚0 ÊHŒ¥ ‚(: ª†ùÓÉá‰ã" proc.memory.rssMB‚ 2383.8046875* proc.memory.percent‚14.549589157104492 cpu‚26.86532058013893 proc.cpu.threads‚17 network.sent‚ 4.860257e+06 network.recv‚ 2.024604e+06% disk./.usageGB‚450.85865020751953! disk.nvme0c0n1.in‚ 314.1640625 disk.sda.in‚ 9434.2578125 disk.sda.out‚ 20.3671875 disk.sda1.out‚0 disk.dm-0.in‚ 314.1640625 disk.sda3.in‚ 9434.2578125 disk.sda3.out‚ 20.3671875 disk.sda1.in‚0 disk.sda2.in‚0% disk.nvme0n1.out‚9837.56884765625 disk.dm-1.in‚ 9434.2578125" disk.dm-0.out‚9837.56884765625$ memory_percent‚65.86766242980957' disk.nvme0c0n1.out‚9837.56884765625 disk.sda2.out‚0 disk.nvme0n1.in‚ 314.1640625 disk.drbd2165.in‚0( proc.memory.availableMB‚ 5592.2421875* disk./.usagePercent‚30.257502311031214 disk.dm-1.out‚ 20.3671875 disk.drbd2165.out‚0”èd0¦ + _wandbruntime‚980  _runtime‚980…áŸsŧ ¿ trainer/global_step‚3649 $ _timestamp‚1.786659632327681e+09 _runtime‚ 980.857735883 _step‚297 & lr-AdamW/pg1‚0.0014005783376884692 & lr-AdamW/pg2‚0.0014005783376884692¶Î²àʨ Ä _step‚297 & lr-AdamW/pg1‚0.0014005783376884692 & lr-AdamW/pg2‚0.0014005783376884692 trainer/global_step‚3649 $ _timestamp‚1.786659632327681e+09 _runtime‚ 980.857735883©ØõÜ\0© + _wandbruntime‚980  _runtime‚980bV¦'ª  O IDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 VEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 882589016576 JBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚665 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚322 M@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚59049984 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 4552982528 XEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚99897142478762 _runtime‚ 980.861898864 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚12254364 KBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚4876 UHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚68876288 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 BDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚4287 XEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚99605548236800 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4289 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 122306526 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚3071051 F?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚60 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚13516123 _LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚99016349220864 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 5494538240 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 _LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚99605565341696 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 5494538240 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚322 IDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4876 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 122905600 YHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 882589016576 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚100488196303360 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 4654602240 D;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚5264 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚13516125 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚100488137253376 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚100488137253376 ZIDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 882630961664 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚262 trainer/global_step‚3649 dQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚86830277459968 PCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚58387380 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚13516125 ]JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚87763685931008 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚20643840 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 [HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚99016332181504 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚4287 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚11485587 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 _step‚298 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 125829120 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 4654602240 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚11485666 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚48232448 IBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚60 IBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚79 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 4517085184 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0ƒEþÕ« ¥O WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17104896 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚59395584 F>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 VEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 882589016576 JBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚665 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚322 M@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚59049984 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 4552982528 XEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚99897142478762 _runtime‚ 980.861898864 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚12254364 KBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚4876 UHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚68876288 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 BDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚4287 XEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚99605548236800 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4289 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 122306526 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚3071051 F?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚60 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚13516123 _LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚99016349220864 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 5494538240 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 _LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚99605565341696 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 5494538240 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚322 IDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4876 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 122905600 YHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 882589016576 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚100488196303360 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 4654602240 D;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚5264 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚13516125 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚100488137253376 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚100488137253376 ZIDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 882630961664 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚262 trainer/global_step‚3649 dQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚86830277459968 PCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚58387380 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚13516125 ]JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚87763685931008 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚20643840 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 [HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚99016332181504 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚4287 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚11485587 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 _step‚298 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 125829120 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 4654602240 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚11485666 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚48232448 IBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚60 IBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚79 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 4517085184 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 IDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 D6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚322 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 122905600 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 ]JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚99042535014400 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚262 PDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654602240 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚25777335 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚59395584 NADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚58387428 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17104896 NADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚59053056 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 4552982528 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 bODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚86853324832768 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 125829120 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚12257657 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚11488596 VCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚99923558844958 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 5494538240 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 5368709120 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚13519678 trainer/global_step‚3649 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚8416818 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚223 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚322 YFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚99631870574592 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 122905600 EDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521206 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 125829120 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 $ _timestamp‚1.786659632496494e+09 >5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5264 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚100514749712896 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚41948160 G@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚72 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚44035072 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚322 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 _runtime‚ 981.026855537 H@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚665 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚3071848 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 4654602240 ^KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚86853304188928 ZGDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚99631887679488 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚100514690659840 WFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 882820085248 \KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 933607890944 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚13519678 K>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59053056 [JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 882862033408 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚12253364 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚12257657 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 845178880 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 5494538240 _step‚299 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚3071778 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 4517085184 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚41348068 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17104896 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚25773040 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚13519676 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚429 BDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 D6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚322 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 122905600 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 ]JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚99042535014400 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚262 PDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654602240 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚25777335 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚59395584 NADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚58387428 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17104896 NADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚59053056 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 4552982528 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 bODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚86853324832768 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 125829120 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚12257657 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚11488596 VCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚99923558844958 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 5494538240 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 5368709120 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚13519678 trainer/global_step‚3649 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚8416818 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚223 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚322 YFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚99631870574592 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 122905600 EDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521206 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 125829120 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 $ _timestamp‚1.786659632496494e+09 >5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5264 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚100514749712896 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚41948160 G@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚72 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚44035072 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚322 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 _runtime‚ 981.026855537 H@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚665 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚3071848 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 4654602240 ^KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚86853304188928 ZGDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚99631887679488 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚100514690659840 WFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 882820085248 \KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 933607890944 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚13519678 K>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59053056 [JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 882862033408 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚12253364 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚12257657 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 845178880 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 5494538240 _step‚299 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚3071778 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 4517085184 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚41348068 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17104896 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚25773040 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚13519676 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚429 Bŵ ¿ trainer/global_step‚3699 % _timestamp‚1.7866596456721408e+09 _runtime‚ 994.20222861 _step‚301 & lr-AdamW/pg1‚0.0013975668791261896 & lr-AdamW/pg2‚0.0013975668791261896eµ~«Ê¶ Ä & lr-AdamW/pg1‚0.0013975668791261896 & lr-AdamW/pg2‚0.0013975668791261896 trainer/global_step‚3699 % _timestamp‚1.7866596456721408e+09 _runtime‚ 994.20222861 _step‚301­„õ;0· + _wandbruntime‚994  _runtime‚994¿j\yþ¸ ¨O JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚4287 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚101815840908800 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚41945088 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚4287 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚8524865 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚3111891 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚209 PCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚58387380 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚13693775 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚44038144 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 E8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚26108952 @7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚5264 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚101815899958784 YHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 894175782400 KDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚60 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚11636695 G>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4876 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚13693773 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 4654602240 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚100921682231296 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚11636756 YHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 892313720486 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 5494538240 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚100921682231296 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚101217935578790 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 125829120 F?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚60 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17104896 PCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚59049984 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚223 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚262 IDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚262 KBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚4876 trainer/global_step‚3699 M@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚59049984 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚26113241 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 831258624 UHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚64681984 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚100921665126400 _runtime‚ 994.206925724 C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚4289 F>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 JBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚665 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚101815840908800 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 5494538240 ]LDeviceStatsMonitor.on_tFG—­°rain_batch_start/allocated_bytes.small_pool.allocated‚ 894217727488 dQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚87982646099968 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 125829120 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚429 PIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚59 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 ]LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 892355068506 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚41348020 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 5368709120 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 125829120 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17104896 IBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚61 YFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚88928243614720 `MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚87982625456128 ^MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 945618158592 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 4517085184 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚262 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚13693773 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚41945088 VEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 894175782400 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚100325621858304 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 IBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚60 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚12419466 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚12415179 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 122306526 % _timestamp‚1.7866596456732283e+09 _step‚302 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 5368709120 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚12415179 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4289 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚3111832 BDeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4289 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚3111832 BDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚101815840908800 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚41945088 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚4287 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚8524865 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚3111891 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚209 PCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚58387380 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚13693775 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚44038144 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 E8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚26108952 @7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚5264 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚101815899958784 YHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 894175782400 KDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚60 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚11636695 G>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4876 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚13693773 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 4654602240 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚100921682231296 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚11636756 YHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 892313720486 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 5494538240 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚100921682231296 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚101217935578790 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 125829120 F?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚60 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17104896 PCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚59049984 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚223 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚262 IDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚262 KBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚4876 trainer/global_step‚3699 M@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚59049984 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚26113241 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 831258624 UHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚64681984 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚100921665126400 _runtime‚ 994.206925724 C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚4289 F>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 JBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚665 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚101815840908800 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 5494538240 ]LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 894217727488 dQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚87982646099968 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360®ž‚ T0º + _wandbruntime‚994  _runtime‚994?ô¸&» ²M WFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 894406851072 E8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚101842453368320 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚59395584 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 5368709120 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚26120087 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚44035072 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚101244351944938 [JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 892585641166 XGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 894448799232 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚3112699 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚100351807651840 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 5494538240 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 5368709120 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 125829120 PDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521206 D=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚60 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚13697328 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17104896 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17104896 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚3112638 [HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚88951602188288 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚100948004569088 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 845178880 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚41348068 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 DDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59053056 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚100947987464192 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 4552982528 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚262 >5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5264 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 bODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚88005693472768 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚20643840 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 G@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚60 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚100947987464192 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚12422759 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚322 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚262 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚4293 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚12422759 NADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚59053056 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚322 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚41948160 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 122905600 WDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚88951537509376 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 5494538240 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚209 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 5494538240 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚262 EDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚11639706 \KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 945864680448 BDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚4293 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚41948160 [JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 894448799232 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚26115792 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚101244410332366 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚101842453368320 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚11639769 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚8527068 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 4552982528 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚13697328 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚13697326 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚429 SFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚64678912 ^KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚88005672828928 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚100351824691200 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 125829120 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 % _timestamp‚1.7866596458561554e+09 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654602240 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚12418466 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚26120087 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚100948004569088 G@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚63 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 831258624 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 NADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚58387428 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚12418466 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚101842394315264 NGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚61 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚322 IBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚60 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚26115792 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚13697326 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 4654602240 TCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 894406851072 H@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚665 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 trainer/global_step‚3699 _runtime‚ 994.386665895 `ODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 945908715520 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 WFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 892544293098¸>¶ç’¼ ·M SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 122306526 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 5368709120 _step‚303 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚4295 K>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59053056 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚100947987464192 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 4552982528 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚262 >5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5264 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 bODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚88005693472768 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚20643840 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 G@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚60 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚100947987464192 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚12422759 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚322 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚262 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚4293 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚12422759 NADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚59053056 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚322 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚41948160 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 122905600 WDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚88951537509376 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 5494538240 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚209 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 5494538240 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚262 EDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚11639706 \KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 945864680448 BDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚4293 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚41948160 [JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 894448799232 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚26115792 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚101244410332366 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚101842453368320 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚11639769 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚8527068 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 4552982528 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚13697328 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚13697326 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚429 SFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚64678912 ^KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚88005672828928 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚100351824691200 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 125829120 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 % _timestamp‚1.7866596458561554e+09 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654602240 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚12418466 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚26120087 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚100948004569088 G@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚63 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 831258624 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 NADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚58387428 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚12418466 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚101842394315264 NGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚61 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚322 IBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚60 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚26115792 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚13697326 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 4654602240 TCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 894406851072 H@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚665 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 trainer/global_step‚3699 _runtime‚ 994.386665895 `ODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 945908715520 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 WFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 892544293098 WFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 894406851072 E8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚101842453368320 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚59395584 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 5368709120 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚26120087 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚44035072 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚101244351944938 [JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 892585641166 XGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 894448799232 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚3112699 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚100351807651840 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 5494538240 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 5368709120 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 125829120 PDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521206 D=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚60 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚13697328 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17104896 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17104896 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚3112638 [HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚88951602188288 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚100948004569088 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 845178880 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚41348068 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 D™2 - _wandbruntime‚1007  _runtime‚1007LxVÇà Á _step‚305 & lr-AdamW/pg1‚0.0013945138285533418 & lr-AdamW/pg2‚0.0013945138285533418 trainer/global_step‚3749 % _timestamp‚1.7866596590338037e+09 _runtime‚1007.565926866Ã>ÊÌÄ Æ & lr-AdamW/pg1‚0.0013945138285533418 & lr-AdamW/pg2‚0.0013945138285533418 trainer/global_step‚3749 % _timestamp‚1.7866596590338037e+09 _runtime‚1007.565926866 _step‚305±§†Tƒ2Å - _wandbruntime‚1007  _runtime‚1007ËÒã¯'Æ ©O ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚102237799120896 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 PIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚57 YFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚90092865348608 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚12580281 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚4287 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚102237782016000 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚322 BDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚102237782016000 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚322 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚59395584 M@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚59049984 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚103143544564224 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚41945088 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚102237799120896 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17104896 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4289 ]JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚90092927933440 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚101634928574464 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 122306526 @7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚5264 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚12584568 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚13871425 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚429 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚3152527 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 _step‚306 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 4654602240 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚20643840 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚13871423 `MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚89134994096128 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 5368709120 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚322 F?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚60 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚13871425 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 4654602240 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚13871423 YHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 903875531094 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 122905600 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚223 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 D;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚5264 IDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4876 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 4552982528 VEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 905762548224 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 831258624 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚262 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 4628521206 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚101634911535104 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 125829120 E8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚26451704 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 122905600 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚41945088 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚11787640 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚8635113 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚262 trainer/global_step‚3749 % _timestamp‚1.7866596590346017e+09 YHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 905762548224 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚8635115 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 125829120 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚103143603614208 ]LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 905804493312 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚11787699 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚322 KDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚60 ZIDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 905804493312 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚103143603614208 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 5494538240¡KR´'Ç ®O D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚101634928574464 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 122306526 @7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚5264 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚12584568 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚13871425 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚429 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚3152527 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 _step‚306 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 4654602240 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚20643840 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚13871423 `MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚89134994096128 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 5368709120 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚322 F?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚60 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚13871425 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 4654602240 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚13871423 YHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 903875531094 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 122905600 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚223 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 D;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚5264 IDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4876 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 4552982528 VEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 905762548224 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 831258624 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚262 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 4628521206 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚101634911535104 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 125829120 E8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚26451704 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 122905600 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚41945088 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚11787640 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚8635113 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚262 trainer/global_step‚3749 % _timestamp‚1.7866596590346017e+09 YHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 905762548224 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚8635115 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 125829120 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚103143603614208 ]LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 905804493312 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚11787699 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚322 KDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚60 ZIDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 905804493312 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚103143603614208 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 5494538240 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚102237799120896 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 PIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚57 YFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚90092865348608 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚12580281 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚4287 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚102237782016000 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚322 BDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚102237782016000 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚322 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚59395584 M@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚59049984 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚103143544564224 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚41945088 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚102237799120896 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17104896 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4289 ]JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚90092927933440 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0²%rý2È - _wandbruntime‚1007  _runtime‚10072çm5®É ³M WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚103170157023744 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚13874978 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654602240 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 5368709120 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 4552982528 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚41348068 >5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5264 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚103170097970688 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 4552982528 [HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚90116219613184 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚12583568 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚12583568 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚26458544 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚41948160 \KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 958113465344 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚8637318 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17104896 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚102565203432346 NADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚59053056 [JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 906035565056 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚26462839 H@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚665 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚26462839 EDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59053056 P6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚322 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚262 NGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚59 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚223 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚12587861 I@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚4876 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚101661097328640 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 125829120 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 845178880 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 122306526 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚262 WFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 905993616896 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚262 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚4295 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚41948160 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚3153417 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 5368709120 % _timestamp‚1.7866596592228022e+09 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚322 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 ^KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚89158041468928 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 125829120 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 125829120 D8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 TCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 905993616896 WDDeviceStat?kþD sMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚90116154934272 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚44035072 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 5494538240 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 122905600 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚4293 G@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚61 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚101661114368000 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 5494538240 _runtime‚1007.753350901 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 4654602240 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚102264121458688 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚209 SFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚64678912 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 4517085184 trainer/global_step‚3749 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 831258624 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚102264104353792 B9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚5264 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚13874976 `ODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 958157500416 D=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚60 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚102264104353792 NADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚58387428 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚322 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚322 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚13874978 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚13874976 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚12587861 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚4293 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521206 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 _step‚307 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚11790676 B[ñß¾&Ê ¸M XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 5368709120 % _timestamp‚1.7866596592228022e+09 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚322 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 ^KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚89158041468928 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 125829120 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 125829120 D8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 TCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 905993616896 WDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚90116154934272 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚44035072 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 5494538240 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 122905600 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚4293 G@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚61 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚101661114368000 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 5494538240 _runtime‚1007.753350901 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 4654602240 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚102264121458688 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚209 SFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚64678912 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 4517085184 trainer/global_step‚3749 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 831258624 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚102264104353792 B9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚5264 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚13874976 `ODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 958157500416 D=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚60 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚102264104353792 NADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚58387428 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚322 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚322 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚13874978 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚13874976 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚12587861 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚4293 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521206 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 _step‚307 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚11790676 BDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654602240 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 5368709120 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 4552982528 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚41348068 >5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5264 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚103170097970688 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 4552982528 [HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚90116219613184 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚12583568 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚12583568 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚26458544 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚41948160 \KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 958113465344 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚8637318 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17104896 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚102565203432346 NADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚59053056 [JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 906035565056 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚26462839 H@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚665 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚26462839 EDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59053056 P6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚322 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚262 NGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚59 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚223 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚12587861 I@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚4876 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚101661097328640 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 125829120 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 845178880 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 122306526 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚262 WFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 905993616896 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚262 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚4295 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚41948160 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚3153417³Q·2Ë - _wandbruntime‚1007  _runtime‚1007˜t’rÌ ì 0loss/FRt1/assignment_loss‚0.8392516995404042 0loss/FRt2/assignment_loss‚0.7736411625510103 /loss/FRt2/detection_loss‚0.5049544740249191 epoch‚9 trainer/global_step‚3749 _runtime‚1007.753888289 _step‚308 /loss/FRt1/detection_loss‚0.5771849987041107 &loss/total_loss‚2.6950323348204446 % _timestamp‚1.7866596592240536e+09Z*wÍ ñ 0loss/FRt1/assignment_loss‚0.8392516995404042 0loss/FRt2/assignment_loss‚0.7736411625510103 /loss/FRt2/detection_loss‚0.5049544740249191 epoch‚9 trainer/global_step‚3749 _runtime‚1007.753888289 _step‚308 /loss/FRt1/detection_loss‚0.5771849987041107 &loss/total_loss‚2.6950323348204446 % _timestamp‚1.7866596592240536e+09´º»>ª Î ‚(:• ׆ùÓ×±£á gpu.0.gpu‚51.0 gpu.0.memory‚9, gpu.0.memoryAllocated‚7.0313262939453125) gpu.0.memoryAllocatedBytes‚ 6039863296 gpu.0.temp‚36.0 gpu.0.powerWatts‚212.489' gpu.0.enforcedPowerLimitWatts‚500.0) gpu.0.powerPercent‚42.497800000000005 gpu.0.smClock‚1410 gpu.0.memoryClock‚1593$ gpu.0.correctedMemoryErrors‚1955# gpu.0.uncorrectedMemoryErrors‚0*kûÓÏ ‚(:ˆ ׆ùÓÙ¢Ãã disk.dm-1.in‚ 9434.2578125 disk.dm-1.out‚ 20.87890625 disk.drbd2165.in‚0 disk.drbd2165.out‚0* proc.memory.percent‚14.549684524536133 cpu‚26.874578163409367 network.recv‚ 2.116819e+06" disk.nvme0c0n1.in‚ 316.00390625 disk.dm-0.in‚ 316.00390625" proc.memory.rssMB‚ 2383.8203125 proc.cpu.threads‚17 network.sent‚ 5.08071e+06% disk./.usageGB‚450.63584899902344 disk.sda3.in‚ 9434.2578125' disk.nvme0c0n1.out‚10431.1044921875 disk.sda.in‚ 9434.2578125% disk.nvme0n1.out‚10431.1044921875$ memory_percent‚65.86754322052002 disk.sda3.out‚ 20.87890625 disk.sda.out‚ 20.87890625 disk.sda1.in‚0 disk.sda1.out‚0 disk.sda2.in‚0 disk.nvme0n1.in‚ 316.00390625" disk.dm-0.out‚10431.1044921875) proc.memory.availableMB‚ 5592.26171875* disk./.usagePercent‚30.242549934986375 disk.sda2.out‚0‰“’m2Ð - _wandbruntime‚1020  _runtime‚1020˜„ùÅÑ ¿ _runtime‚1020.306031892 _step‚309 % lr-AdamW/pg1‚0.001391419382061658 % lr-AdamW/pg2‚0.001391419382061658 trainer/global_step‚3799 % _timestamp‚1.7866596717759552e+09ÙéÎÊÒ Ä _step‚309 % lr-AdamW/pg1‚0.001391419382061658 % lr-AdamW/pg2‚0.001391419382061658 trainer/global_step‚3799 % _timestamp‚1.7866596717759552e+09 _runtime‚1020.306031892µýä '2Ó - _wandbruntime‚1020  _runtime‚1020SÁv·®'Ô ¨O @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚322 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚14049073 E8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚26794456 ]JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚91257561811968 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚103553898905600 D;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚5264 PIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚68 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚44038144 _runtime‚ 1020.31092479 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚14049073 G>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4876 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 5494538240 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 125829120 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚12749670 YFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚91257497129984 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚104471307269632 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 4654602240 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚322 IDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 4628521206 PCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚58387380 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚102944201211904 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 122306526 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 5494538240 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚41945088 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚26798745 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚11938400 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 5494538240 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 M@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚59049984 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 122905600 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚3193105 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 125829120 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚104471248219648 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚103553916010496 JBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚665 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚103859638553606 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚322 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚262 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚14049075 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17104896 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 122905600 YHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 917349314048 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 % _timestamp‚1.7866596717769656e+09 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚4287 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 4552982528 IDeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4289 @7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚5264 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚14049075 ZIDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 917391259136 IBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚70 bQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 970178432000 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚104471307269632 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚4287 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚20643840 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚59395584 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚3193037 ]LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 915478689722 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚262 trainer/global_step‚3799 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚102944218251264 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 5368709120 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚103553916010496 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚12749670 C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚4289 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚8745363 ^MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 970134393856 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚322 F?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚60 F>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0y¬i!íÕ ­O YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚103553898905600 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 4552982528 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 5368709120 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚12745383 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 4654602240 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17104896 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 845178880 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 831258624 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 PCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚59049984 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4289 @7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚5264 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚14049075 ZIDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 917391259136 IBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚70 bQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 970178432000 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚104471307269632 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚4287 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚20643840 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚59395584 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚3193037 ]LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 915478689722 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚262 trainer/global_step‚3799 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚102944218251264 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 5368709120 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚103553916010496 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚12749670 C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚4289 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚8745363 ^MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 970134393856 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚322 F?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚60 F>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚322 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚14049073 E8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚26794456 ]JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚91257561811968 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚103553898905600 D;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚5264 PIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚68 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚44038144 _runtime‚ 1020.31092479 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚14049073 G>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4876 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 5494538240 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 125829120 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚12749670 YFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚91257497129984 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚104471307269632 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 4654602240 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚322 IDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 4628521206 PCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚58387380 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚102944201211904 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 122306526 SDDeviceStatsMonitor.on_train_batch_star?CS†Æ t/reserved_bytes.all.allocated‚ 5494538240 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚41945088 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚26798745 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚11938400 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 5494538240 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 M@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚59049984 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 122905600 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚3193105 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 125829120 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚104471248219648 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚103553916010496 JBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚665 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚103859638553606 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚322 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚262 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚14049075 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17104896 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 122905600 YHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 917349314048 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 % _timestamp‚1.7866596717769656e+09 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚4287 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 4552982528 IDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚11941387 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚103886054919754 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚104497860679168 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521206 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚41348068 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚322 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚12748670 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚14052628 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚12748670 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚20643840 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚59395584 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚26805591 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 5368709120 D=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚60 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚223 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚103580221243392 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17104896 `ODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 970425786368 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 125829120 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚322 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚12752963 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 NADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚59053056 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚14052626 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 5494538240 _step‚311 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚4293 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 125829120 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 125829120 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 trainer/global_step‚3799 K>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59053056 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚14052628 B9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚5264 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚429 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 831258624 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 5368709120 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚104497860679168 SFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚70970368 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚103580221243392 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 845178880 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 WFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 915667914314 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚262 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚12752963 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚104497801626112 WDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚91280785568768 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚322 I@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚4876 >5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5264 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚26801296 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚4293 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚50326528 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚262 H@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚665 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚26805591 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚102970404044800 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚322 IBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚60 EDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 4517085184 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654602240 [HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚91280856539136 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 5368709120 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚26801296 G@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚76 B8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 DDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 4517085184 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654602240 [HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚91280856539136 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 5368709120 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚26801296 G@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚76 B8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 DDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚11941387 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚103886054919754 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚104497860679168 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521206 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚41348068 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚322 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚12748670 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚14052628 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚12748670 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚20643840 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚59395584 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚26805591 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 5368709120 D=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚60 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚223 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚103580221243392 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17104896 `ODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 970425786368 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 125829120 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚322 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚12752963 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 NADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚59053056 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚14052626 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 5494538240 _step‚311 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚4293 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 125829120 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 125829120 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 trainer/global_step‚3799 K>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59053056 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚14052628 B9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚5264 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚429 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 831258624 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 5368709120 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚104497860679168 SFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚70970368 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚103580221243392 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 845178880 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 WFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 915667914314 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚262 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚12752963 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚104497801626112 WDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚91280785568768 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚322 I@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚4876 >5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5264 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚26801296 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚4293 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚50326528 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚262 H@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚665 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚26805591 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚102970404044800 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚322 IBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚60 EDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 YHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 928936079872 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚48232448 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 trainer/global_step‚3849 IDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4876 E8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚27137208 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚105798951875072 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚105798951875072 IDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚14226723 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚8855613 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚322 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚104253490888704 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 122905600 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 4654602240 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4289 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚14226723P»¨h´'á ®O @7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚5264 PCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚58387380 BDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚14226723 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚8855613 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚322 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚104253490888704 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 122905600 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 4654602240 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4289 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚14226723 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚105180548428394 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 5494538240 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 _step‚314 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚105799010925056 PCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚59049984 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚104870015795200 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚41348020 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚4287 ZIDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 928978024960 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚12914772 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚209 dQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚91439752019968 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚59395584 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 F>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 YHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 928936079872 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚48232448 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 trainer/global_step‚3849 IDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4876 E8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚27137208 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚105798951875072 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚105798951875072 IDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521206 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 831258624 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚41948160 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚105825505281536 [JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 929209096704 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚44035072 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚104279676682240 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚4295 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 122905600 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚209 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚105206964794590 trainer/global_step‚3849 WFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 929167148544 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚12918065 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚12913772 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚3234459 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚12913772 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 4552982528 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚41948160 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚104896338132992 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 5494538240 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 5368709120 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 125829120 D=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚60 _runtime‚1034.291322504 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 845178880 WFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 927229724922 % _timestamp‚1.7866596857610059e+09 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚27148343 E5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5264 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚429 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 125829120 G@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚60 DDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚12092277 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚3234521 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 122306526 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚262 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚14230278 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚104279693721600 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚41348068 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 5494538240 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17104896 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 NADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚59053056 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚104896355237888 B9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚5264 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 125829120 TCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 929167148544 H@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚665 NGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚62 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚27144048 PDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 K>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59053056 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚20643840 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 4517085184 XGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 929209096704 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚27144048 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚8857818 SFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚64678912 \KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 982652736512 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚322 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 [JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 927271072990 _step‚315 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚12918065 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚104896355237888 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚8857820 E6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚322 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚322 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 4654602240 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654602240 [HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚92445496164352 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚105825564334592 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚14230278 G@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚64 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚105206906407162 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚262«£)½¾&ä ¸M _step‚315 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚12918065 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚104896355237888 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚8857820 E6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚322 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚322 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 4654602240 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654602240 [HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚92445496164352 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚105825564334592 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚14230278 G@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚64 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚105206906407162 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚262 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚4293 ^KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚91462778748928 NADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚58387428 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521206 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 831258624 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚41948160 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚105825505281536 [JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 929209096704 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚44035072 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚104279676682240 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚4295 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 122905600 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚209 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚105206964794590 trainer/global_step‚3849 WFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 929167148544 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚12918065 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚12913772 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚3234459 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚12913772 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 4552982528 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚41948160 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚104896338132992 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 5494538240 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 5368709120 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 125829120 D=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚60 _runtime‚1034.291322504 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 845178880 WFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 927229724922 % _timestamp‚1.7866596857610059e+09 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚27148343 E5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5264 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚429 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 125829120 G@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚60 DDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚12092277 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚3234521 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 122306526 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚262 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚14230278 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚104279693721600 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚41348068 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 5494538240 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17104896 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 NADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚59053056 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚104896355237888 B9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚5264 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 125829120 TCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 929167148544 H@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚665 NGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚62 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚27144048 PDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 K>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59053056 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚20643840 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 4517085184 XGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 929209096704 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚27144048 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚8857818 SFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚64678912 \KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 982652736512 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚322 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 [JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 927271072990»LõÉ2å - _wandbruntime‚1034  _runtime‚1034~ekbræ ì &loss/total_loss‚2.9011931773625834 epoch‚9 trainer/global_step‚3849 _runtime‚1034.291941092 0loss/FRt1/assignment_loss‚0.9219059485293751 /loss/FRt2/detection_loss‚0.5510458386759332 % _timestamp‚1.7866596857620685e+09 _step‚316 0loss/FRt2/assignment_loss‚0.8795255493323576 /loss/FRt1/detection_loss‚0.54871584082491764ÙD®Cç ñ _step‚316 0loss/FRt2/assignment_loss‚0.8795255493323576 /loss/FRt1/detection_loss‚0.5487158408249176 &loss/total_loss‚2.9011931773625834 epoch‚9 trainer/global_step‚3849 _runtime‚1034.291941092 0loss/FRt1/assignment_loss‚0.9219059485293751 /loss/FRt2/detection_loss‚0.55104583XD?|486759332 % _timestamp‚1.7866596857620685e+09¼d°¡è ‚(:– æ†ùӠ¡á gpu.0.gpu‚64.0 gpu.0.memory‚12, gpu.0.memoryAllocated‚7.0313262939453125) gpu.0.memoryAllocatedBytes‚ 6039863296 gpu.0.temp‚36.0 gpu.0.powerWatts‚153.156' gpu.0.enforcedPowerLimitWatts‚500.0) gpu.0.powerPercent‚30.631200000000003 gpu.0.smClock‚1410 gpu.0.memoryClock‚1593$ gpu.0.correctedMemoryErrors‚1955# gpu.0.uncorrectedMemoryErrors‚0Û¤cw‡é ‚(:ü æ†ùÓÁ±¶ã network.recv‚ 2.143705e+06 disk.sda3.in‚ 9434.2578125 disk.sda1.out‚0$ disk.nvme0n1.out‚10623.611328125 disk.dm-1.in‚ 9434.2578125 disk.drbd2165.out‚0( proc.memory.availableMB‚ 5591.9296875* proc.memory.percent‚14.549684524536133 network.sent‚ 5.150709e+06 disk.sda.in‚ 9434.2578125 disk.sda1.in‚0 disk.sda2.out‚0 disk.dm-1.out‚ 21.296875 disk.dm-0.in‚ 316.00390625$ memory_percent‚65.86956977844238 proc.cpu.threads‚17) disk./.usagePercent‚30.18012096171667$ disk./.usageGB‚449.7056121826172" disk.nvme0c0n1.in‚ 316.00390625& disk.nvme0c0n1.out‚10623.611328125 disk.sda.out‚ 21.296875 disk.nvme0n1.in‚ 316.00390625 disk.drbd2165.in‚0" proc.memory.rssMB‚ 2383.8203125 disk.sda3.out‚ 21.296875 disk.sda2.in‚0! disk.dm-0.out‚10623.611328125 cpu‚26.87727054478717–^"¶2ê - _wandbruntime‚1046  _runtime‚1046þô\JÅë ¿ & lr-AdamW/pg1‚0.0013851070989698631 & lr-AdamW/pg2‚0.0013851070989698631 trainer/global_step‚3899 # _timestamp‚1.78665969841103e+09 _runtime‚1046.941061244 _step‚317¡~îÊì Ä & lr-AdamW/pg1‚0.0013851070989698631 & lr-AdamW/pg2‚0.0013851070989698631 trainer/global_step‚3899 # _timestamp‚1.78665969841103e+09 _runtime‚1046.941061244 _step‚317½mŸH¬2í - _wandbruntime‚1046  _runtime‚1046|eCK¯'î ©O PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚13075587 KBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚4876 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚223 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 845178880 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚8965865 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚106186149789696 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17104896 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4289 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚429 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 5368709120 % _timestamp‚1.7866596984119844e+09 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚209 IDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4876 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚8965863 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 IBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚60 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 _runtime‚1046.945860874 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17104896 ]LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 940564790784 JBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚665 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 ]JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚93586833256448 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚14404373 UHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚68876288 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 4552982528 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚106501341528422 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚262 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚107126714580480 VEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 940522845696 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 4552982528 PIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚77 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 122306526 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 5494538240 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚322 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚106186132684800 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚41945088 PCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚59049984 ^MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 994664364032 PCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚58387380 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚107126714580480 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 dQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚92592120659968 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚59395584 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 125829120 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 F?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚60 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚13075587 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚12240201 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚107126655530496 M@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚59049984 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚14404375 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 831258624 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 5368709120 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚322 `MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚92592100016128 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚48232448 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚4287 BDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚13079874 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 4654602240 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚106186149789696 IBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚79 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚3274415 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 5368709120 E8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚27479960 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 4654602240 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 4628521206 KDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚60 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 _step‚318 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 YFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚93586764380160 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚322 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚262 trainer/global_step‚3899 F>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 bQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 994712596480 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚322 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚14404373 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚12240280 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚106501399915802 ]LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 938602310938 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 125829120 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚4289 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚107126655530496 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 122905600 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚4287 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 4517085184 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 125829120 ZIDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 940564790784 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚106186132684800 YHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 940522845696 D;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚5264 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚262 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚105562780565504 YHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 938560962918àúžÔ´'ï ®O XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 5368709120 % _timestamp‚1.7866596984119844e+09 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚209 IDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4876 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚8965863 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 IBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚60 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 _runtime‚1046.945860874 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17104896 ]LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 940564790784 JBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚665 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 ]JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚93586833256448 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚14404373 UHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚68876288 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 4552982528 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚106501341528422 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚262 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚107126714580480 VEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 940522845696 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 4552982528 PIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚77 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 122306526 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 5494538240 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚322 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚106186132684800 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚41945088 PCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚59049984 ^MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 994664364032 PCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚58387380 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚107126714580480 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 dQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚92592120659968 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚59395584 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 125829120 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 F?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚60 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚13075587 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚12240201 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚107126655530496 M@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚59049984 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚14404375 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 831258624 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 5368709120 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚322 `MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚92592100016128 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚48232448 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚4287 BDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚13079874 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 4654602240 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚106186149789696 IBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚79 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚3274415 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 5368709120 E8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚27479960 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 4654602240 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 4628521206 KDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚60 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 _step‚318 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 YFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚93586764380160 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚322 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚262 trainer/global_step‚3899 F>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 bQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 994712596480 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚322 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚14404373 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚12240280 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚106501399915802 ]LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 938602310938 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 125829120 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚4289 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚107126655530496 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 122905600 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚4287 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 4517085184 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 125829120 ZIDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 940564790784 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚106186132684800 YHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 940522845696 D;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚5264 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚262 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚105562780565504 YHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 938560962918 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚13075587 KBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚4876 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚223 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 845178880 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚8965865 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚106186149789696 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17104896 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4289 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚429 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0¾B¼ÛX2ð - _wandbruntime‚1047  _runtime‚1047Ÿt–¸&ñ ²M UIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚8968070 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚223 D6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚322 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚4293 E5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5264 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 4552982528 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 125829120 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚13083167 WFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 940753914368 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚14407928 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚59395584 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚12243294 G@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚73 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚429 `ODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 994956903424 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚322 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17104896 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚27491095 WFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 938791535530 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚322 IBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚60 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚41348068 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 5368709120 _runtime‚1047.104139914 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚107153208936960 SFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚64678912 \KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 994912868352 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚106527757894570 G@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚60 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 122905600 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚4293 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚3275224 [HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚93610124936192 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚322 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 bODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚92615168032768 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚4295 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 122905600 B9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚5264 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 122306526 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 EDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17104896 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚106527816281998 D=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚60 _step‚319 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654602240 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 4552982528 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚262 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚13078874 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚27486800 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚3275153 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 5494538240 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚41948160 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚14407926 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚12243221 ^KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚92615147388928 trainer/global_step‚3899 [JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 940795862528 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521206 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚262 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚27491095 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚14407926 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚8968068 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 5494538240 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚107153267990016 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 WDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚93610060257280 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚20643840 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 831258624 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚105588966359040 BDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚106212472127488 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚106212455022592 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚14407928 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 4654602240 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚106212472127488 K>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59053056 XGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 940795862528 H@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚665 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 5368709120 $ _timestamp‚1.786659698573839e+09 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚27486800 TCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 940753914368–:ô¯ò ·M QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 5494538240 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚107153267990016 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 WDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚93610060257280 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚20643840 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 831258624 Z1SK¢­$FDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚105588966359040 BDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚106212472127488 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚106212455022592 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚14407928 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 4654602240 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚106212472127488 K>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59053056 XGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 940795862528 H@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚665 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 5368709120 $ _timestamp‚1.786659698573839e+09 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚27486800 TCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 940753914368 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚8968070 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚223 D6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚322 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚4293 E5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5264 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 4552982528 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 125829120 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚13083167 WFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 940753914368 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚14407928 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚59395584 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚12243294 G@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚73 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚429 `ODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 994956903424 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚322 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17104896 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚27491095 WFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 938791535530 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚322 IBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚60 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚41348068 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 5368709120 _runtime‚1047.104139914 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚107153208936960 SFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚64678912 \KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 994912868352 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚106527757894570 G@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚60 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 122905600 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚4293 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚3275224 [HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚93610124936192 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚322 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 bODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚92615168032768 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚4295 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 122905600 B9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚5264 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 122306526 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 EDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17104896 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚106527816281998 D=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚60 _step‚319 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654602240 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 4552982528 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚262 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚13078874 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚27486800 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚3275153 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 5494538240 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚41948160 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚14407926 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚12243221 ^KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚92615147388928 trainer/global_step‚3899 [JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 940795862528 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521206 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚262 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚27491095 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚14407926 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚8968068¿6ËB2ó - _wandbruntime‚1047  _runtime‚10473\ôrô ì _runtime‚1047.104667521 0loss/FRt1/assignment_loss‚0.8236941698237527 /loss/FRt2/detection_loss‚0.4555708395186729 &loss/total_loss‚2.5602208688925754 epoch‚9 trainer/global_step‚3899 _step‚320 0loss/FRt2/assignment_loss‚0.6888188562913614 /loss/FRt1/detection_loss‚0.5921370032587885 % _timestamp‚1.7866596985748773e+09p¡üwõ ñ 0loss/FRt1/assignment_loss‚0.8236941698237527 /loss/FRt2/detection_loss‚0.4555708395186729 &loss/total_loss‚2.5602208688925754 epoch‚9 trainer/global_step‚3899 _step‚320 0loss/FRt2/assignment_loss‚0.6888188562913614 /loss/FRt1/detection_loss‚0.5921370032587885 % _timestamp‚1.7866596985748773e+09 _runtime‚1047.104667521Àv'Ž–ö ‚(:‹ õ†ùÓÍ©ôà gpu.0.gpu‚55.0 gpu.0.memory‚10, gpu.0.memoryAllocated‚7.0313262939453125) gpu.0.memoryAllocatedBytes‚ 6039863296 gpu.0.temp‚36.0 gpu.0.powerWatts‚213.059' gpu.0.enforcedPowerLimitWatts‚500.0 gpu.0.powerPercent‚42.6118 gpu.0.smClock‚1410 gpu.0.memoryClock‚1593$ gpu.0.correctedMemoryErrors‚1955# gpu.0.uncorrectedMemoryErrors‚0v­“÷ ‚(:ˆ õ†ùÓÜÓ£ã disk.dm-1.out‚ 21.5234375 disk.sda3.in‚ 9434.2578125( disk.nvme0c0n1.out‚12675.42919921875 disk.sda1.in‚0 disk.sda2.in‚0 network.sent‚ 5.246164e+06 disk.drbd2165.in‚0 disk.dm-0.in‚ 316.00390625" proc.memory.rssMB‚ 2384.6171875 proc.cpu.threads‚17" disk.nvme0c0n1.in‚ 316.00390625 disk.sda.out‚ 21.5234375 disk.sda2.out‚0$ memory_percent‚65.87321758270264) proc.memory.availableMB‚ 5591.33203125* proc.memory.percent‚14.554548263549805 cpu‚26.880121763321984 disk.sda3.out‚ 21.5234375* disk./.usagePercent‚30.301930626545875$ disk./.usageGB‚451.5206642150879& disk.nvme0n1.out‚12675.42919921875 disk.drbd2165.out‚0# disk.dm-0.out‚12675.42919921875 disk.sda.in‚ 9434.2578125 disk.sda1.out‚0 network.recv‚ 2.177051e+06 disk.nvme0n1.in‚ 316.00390625 disk.dm-1.in‚ 9434.2578125ã½2ø - _wandbruntime‚1059  _runtime‚1059*_yXÇù Á _runtime‚1059.504887796 _step‚321 & lr-AdamW/pg1‚0.0013818896677958998 & lr-AdamW/pg2‚0.0013818896677958998 trainer/global_step‚3949 % _timestamp‚1.7866597109729266e+09c†Ù˜Ìú Æ & lr-AdamW/pg1‚0.0013818896677958998 & lr-AdamW/pg2‚0.0013818896677958998 trainer/global_step‚3949 % _timestamp‚1.7866597109729266e+09 _runtime‚1059.504887796 _step‚321Á—r$Ë2û - _wandbruntime‚1059  _runtime‚1059ç+!±'ü «O WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17104896 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 ]LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 950164121546 trainer/global_step‚3949 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚41945088 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚12391042 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚106872087281664 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚322 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚27827001 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚14582025 PIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚86 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 1006990191104 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚41348020 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚14582023 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚41945088 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 125829120 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 4552982528 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 845178880 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚59395584 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚322 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 5368709120 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 4654602240 IDeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4289 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 YHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 950122773526 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚4287 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚107502266679296 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 4552982528 ]LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 952151556608 PCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚58387380 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚262 M@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚59049984 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚20643840 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 BDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 4654602240 VEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 952109611520 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 125829120 G>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4876 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚108454418235904 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚50329600 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚106872070242304 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 5368709120 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚13240689 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 1006939861504 _runtime‚1059.504967346 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 4517085184 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚322 YHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 952109611520 UHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚70973440 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 % _timestamp‚1.7866597109736075e+09 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚107502266679296 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚13240689 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚12390954 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚262 D;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚5264 dQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚93744489299968 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 125829120 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚14582023 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚429 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚209 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚322 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚13244976 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 122905600 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚3314927 ]JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚94751479491072 IBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚60 YFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚94751408517632 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 5494538240 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚108454418235904 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 4628521206 KDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚60 _step‚322 KBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚4876 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚108454359185920 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17104896 PCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚59049984 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚9076115 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚3314841 `MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚93744468656128 E8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚27822712 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚107502249574400 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚107502249574400 IBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚88 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚9076113 F?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚60 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚223 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 831258624 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚4287 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 122306526 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 JBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚665 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 5494538240 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 IDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚108454359185920 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 5494538240 C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚4289 @7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚5264§6ÀE¶'ý °O `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚106872087281664 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚322 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚27827001 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚14582025 PIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚86 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 1006990191104 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚41348020 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚14582023 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚41945088 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 125829120 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 4552982528 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 845178880 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚59395584 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚322 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 5368709120 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 4654602240 IDeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4289 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 YHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 950122773526 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚4287 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚107502266679296 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 4552982528 ]LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 952151556608 PCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚58387380 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚262 M@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚59049984 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚20643840 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 BDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 4654602240 VEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 952109611520 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 125829120 G>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4876 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚108454418235904 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚50329600 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚106872070242304 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 5368709120 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚13240689 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 1006939861504 _runtime‚1059.504967346 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 4517085184 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚322 YHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 952109611520 UHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚70973440 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 % _timestamp‚1.7866597109736075e+09 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚107502266679296 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚13240689 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚12390954 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚262 D;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚5264 dQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚93744489299968 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 125829120 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚14582023 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚429 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚209 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚322 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚13244976 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 122905600 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚3314927 ]JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚94751479491072 IBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚60 YFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚94751408517632 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 5494538240 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚108454418235904 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 4628521206 KDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚60 _step‚322 KBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚4876 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚108454359185920 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17104896 PCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚59049984 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚9076115 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚3314841 `MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚93744468656128 E8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚27822712 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚107502249574400 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚107502249574400 IBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚88 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚9076113 F?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚60 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚223 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 831258624 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚4287 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 122306526 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 JBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚665 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 5494538240 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 IDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚108454359185920 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 5494538240 C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚4289 @7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚5264 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17104896 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 ]LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 950164121546 trainer/global_step‚3949 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚41945088 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚12391042 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 ë2þ - _wandbruntime‚1059  _runtime‚1059þpc9ÿ µM OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚12394013 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 845178880 ^KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚93767516028928 C;DeviceStatsMoniºŒUž%tor.on_train_batch_end/segment.all.allocated‚322 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚322 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚9078320 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚46132224 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚322 SFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚66776064 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654602240 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚106898256035840 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚13248269 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚20643840 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 4654602240 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 122905600 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚13243976 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚12393941 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚209 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 122306526 % _timestamp‚1.7866597111408644e+09 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚14585576 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚108480971645440 B5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5264 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 4552982528 [JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 950394694206 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 5494538240 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚262 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚27833847 IBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚60 D=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚60 _step‚323 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 1007234334208 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚4295 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚27829552 I@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚4876 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 EDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚27833847 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 [HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚94774771006976 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚59395584 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 5494538240 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚107528589017088 B9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚5264 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚223 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚41348068 WFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 950353346138 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 5494538240 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17104896 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚14585578 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚106898273075200 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 5368709120 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚41948160 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚107528589017088 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚41948160 EDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚262 TCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 952340680192 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 H@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚665 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚3315623 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚13248269 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚27829552 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚14585576 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚429 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 1007188201984 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 5368709120 trainer/global_step‚3949 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 125829120 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚322 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚108480912592384 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚107528571912192 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 5368709120 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚4293 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17104896 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 125829120 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚262 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚14585578 G@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚72 NADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚58387428 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚108480971645440 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚107528571912192 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 122905600 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚9078318 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 G@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚60 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚4293 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 831258624 _runtime‚1059.671232705 DDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59053056 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚3315693 NGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚70 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚107848609381978 XGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 952382628352 NADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚59053056 bODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚93767536672768 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚107848667769406 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚13243976 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 [JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 952382628352 WFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 952340680192 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 125829120 PDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521206 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 4517085184ÈÉ5À&€ ºM H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚262 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚14585578 G@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚72 NADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚58387428 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚108480971645440 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚107528571912192 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 122905600 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚9078318 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 G@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚60 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚4293 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 831258624 _runtime‚1059.671232705 DDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59053056 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚3315693 NGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚70 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚107848609381978 XGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 952382628352 NADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚59053056 bODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚93767536672768 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚107848667769406 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚13243976 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 [JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 952382628352 WFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 952340680192 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 125829120 PDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521206 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 4517085184 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚12394013 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 845178880 ^KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚93767516028928 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚322 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚322 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚9078320 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚46132224 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚322 SFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚66776064 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654602240 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚106898256035840 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚13248269 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚20643840 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 4654602240 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 122905600 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚13243976 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚12393941 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚209 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 122306526 % _timestamp‚1.7866597111408644e+09 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚14585576 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚108480971645440 B5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5264 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 4552982528 [JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 950394694206 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 5494538240 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚262 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚27833847 IBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚60 D=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚60 _step‚323 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 1007234334208 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚4295 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚27829552 I@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚4876 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 EDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚27833847 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 [HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚94774771006976 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚59395584 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 5494538240 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚107528589017088 B9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚5264 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚223 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚41348068 WFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 950353346138 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 5494538240 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17104896 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚14585578 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚106898273075200 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 5368709120 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚41948160 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚107528589017088 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚41948160 EDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚262 TCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 952340680192 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 H@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚665 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚3315623 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚13248269 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚27829552 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚14585576 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚429 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 1007188201984 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 5368709120 trainer/global_step‚3949 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 125829120 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚322 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚108480912592384 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚107528571912192 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 5368709120 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚4293 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17104896 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 125829120Ãònï2 - _wandbruntime‚1059  _runtime‚1059€/PÇp‚ ê &loss/total_loss‚2.8161078857417623 % _timestamp‚1.7866597111419733e+09 _runtime‚1059.671807111 _step‚324 /loss/FRt1/assignment_loss‚0.917493627198969 epoch‚9 trainer/global_step‚3949 0loss/FRt2/assignment_loss‚0.8003925020233892 /loss/FRt1/detection_loss‚0.5704925786645971 .loss/FRt2/detection_loss‚0.527729177854807ê/0uƒ ï trainer/global_step‚3949 0loss/FRt2/assignment_loss‚0.8003925020233892 /loss/FRt1/detection_loss‚0.5704925786645971 .loss/FRt2/detection_loss‚0.527729177854807 &loss/total_loss‚2.8161078857417623 % _timestamp‚1.7866597111419733e+09 _runtime‚1059.671807111 _step‚324 /loss/FRt1/assignment_loss‚0.917493627198969 epoch‚9Äèx|§–„ ‚(:‹ „‡ùÓ±Áá gpu.0.gpu‚64.0 gpu.0.memory‚12, gpu.0.memoryAllocated‚7.0313262939453125) gpu.0.memoryAllocatedBytes‚ 6039863296 gpu.0.temp‚35.0 gpu.0.powerWatts‚225.712' gpu.0.enforcedPowerLimitWatts‚500.0 gpu.0.powerPercent‚45.1424 gpu.0.smClock‚1410 gpu.0.memoryClock‚1593$ gpu.0.correctedMemoryErrors‚1955# gpu.0.uncorrectedMemoryErrors‚0¬@Ì‹… ‚(:€ „‡ùÓ®ôÃã disk.sda1.out‚0 disk.dm-1.in‚ 9477.98828125 disk.drbd2165.in‚0 disk.nvme0c0n1.in‚ 326.421875 disk.sda.in‚ 9477.98828125 disk.sda.out‚ 21.82421875" proc.memory.rssMB‚ 2384.6171875 cpu‚26.88287507415506* disk./.usagePercent‚30.344627307098936% disk./.usageGB‚452.15687561035156 disk.sda2.out‚0$ disk.nvme0n1.out‚12724.974609375& disk.nvme0c0n1.out‚12724.974609375$ memory_percent‚65.87648391723633* proc.memory.percent‚14.554548263549805 disk.dm-1.out‚ 21.82421875 disk.drbd2165.out‚0 disk.sda3.in‚ 9477.98828125' proc.memory.availableMB‚ 5590.796875 proc.cpu.threads‚17 network.sent‚ 5.321245e+06 network.recv‚ 2.209883e+06 disk.sda1.in‚0 disk.sda2.in‚0 disk.nvme0n1.in‚ 326.421875 disk.dm-0.in‚ 326.421875! disk.dm-0.out‚12724.974609375 disk.sda3.out‚ 21.82421875GNö°2† - _wandbruntime‚1071  _runtime‚1071ο8ć ¾ _runtime‚ 1071.98233605 _step‚325 % lr-AdamW/pg1‚0.001378631651529356 % lr-AdamW/pg2‚0.001378631651529356 trainer/global_step‚3999 % _timestamp‚1.7866597234523106e+09èeqɈ à trainer/global_step‚3999 % _timestamp‚1.7866597234523106e+09 _runtime‚ 1071.98233605 _step‚325 % lr-AdamW/pg1‚0.001378631651529356 % lr-AdamW/pg2‚0.001378631651529356Å¢â{ 2‰ - _wandbruntime‚1071  _runtime‚1071Îô̯±'Š «O \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚46135296 _runtime‚1071.986872174 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚59395584 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 5494538240 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 125829120 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚108818383568896 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 dQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚94896857939968 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 5368709120 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚13410078 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚13410078 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 845178880 PCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚58387380 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚322 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚14759673 % _timestamp‚1.7866597234532392e+09 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚13405791 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚3355557 UHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚66779136 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 122306526 M@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚59049984 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 125829120 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚108818383568896 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚41945088 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 122905600 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 5368709120 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚262 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚4287 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17104896 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚108818366464000 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17104896 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 1019253617152 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚262 ]LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 963738322432 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4289 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 E8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚28165464 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚9186365 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚14759675 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 831258624 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 125829120 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚109143044503238 YHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 961684584134 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 4654602240 G>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4876 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚3355623 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 IBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚60 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚109782121891328 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚109782062841344 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚108818366464000 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚108181376958464 F?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚60 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚109782062841344 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 4654602240 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 122905600 _step‚326 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚13405791 VEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 963696377344 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚28169753 JBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚665 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚9186363 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 5368709120 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚262 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚4287 ]JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚95916157692416 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚20643840 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚4289 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚14759673 IBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚68 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚322 PCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚59049984 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚322 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 4517085184 KDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚60 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚223 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 5494538240 D;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚5264 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚12541920 @7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚5264 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚14759675 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 4552982528 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚109143102890618 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 1019299752448 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 4628521206 trainer/global_step‚3999 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 4552982528 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚429 ]LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 961725932154 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚41348020 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚322 YFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚95916090913280 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 ZIDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 963738322432 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚109782121891328 BDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚41945088 PIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚66 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚209/™_4m‹ °O ]LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 963738322432 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4289 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 E8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚28165464 WKDeviceStatsMonitor.on_train_batch_start/inactive_sp%n݃I&lit.large_pool.allocated‚9186365 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚14759675 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 831258624 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 125829120 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚109143044503238 YHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 961684584134 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 4654602240 G>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4876 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚3355623 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 IBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚60 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚109782121891328 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚109782062841344 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚108818366464000 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚108181376958464 F?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚60 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚109782062841344 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 4654602240 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 122905600 _step‚326 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚13405791 VEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 963696377344 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚28169753 JBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚665 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚9186363 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 5368709120 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚262 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚4287 ]JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚95916157692416 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚20643840 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚4289 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚14759673 IBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚68 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚322 PCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚59049984 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚322 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 4517085184 KDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚60 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚223 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 5494538240 D;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚5264 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚12541920 @7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚5264 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚14759675 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 4552982528 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚109143102890618 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 1019299752448 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 4628521206 trainer/global_step‚3999 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 4552982528 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚429 ]LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 961725932154 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚41348020 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚322 YFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚95916090913280 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 ZIDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 963738322432 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚109782121891328 BDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚41945088 PIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚66 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚209 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚46135296 _runtime‚1071.986872174 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚59395584 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 5494538240 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 125829120 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚108818383568896 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 dQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚94896857939968 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 5368709120 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚13410078 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚13410078 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 845178880 PCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚58387380 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚322 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚14759673 % _timestamp‚1.7866597234532392e+09 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚13405791 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚3355557 UHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚66779136 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 122306526 M@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚59049984 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 125829120 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚108818383568896 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚41945088 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 122905600 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 5368709120 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚262 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚4287 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17104896 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚108818366464000 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17104896 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 1019253617152 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚262ÆNgpâ2Œ - _wandbruntime‚1072  _runtime‚1072ˆŸ¯º& ´M ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚108207545712640 IBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚60 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚41948160 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚12544927 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 845178880 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚108844688801792 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17104896 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚20643840 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚109808675300864 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 1019546516992 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚262 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚41948160 TCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 963927446016 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 4552982528 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚28172304 NGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚68 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚322 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 [JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 961956504814 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚4293 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚108844688801792 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚12544997 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 ^KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚94919884668928 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚108844705906688 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17104896 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚223 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚46132224 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚322 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 122905600 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚13413371 I@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚4876 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚109169460869386 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚28172304 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚14763228 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚13409078 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚109808616247808 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚14763226 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 125829120 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 122905600 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚109169519256814 _runtime‚1072.144392944 SFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚66776064 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 831258624 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚322 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654602240 B9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚5264 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚9188570 WFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 961915156746 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚322 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 4552982528 [HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚95939451829760 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚209 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚28176599 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 5494538240 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 5494538240 K>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59053056 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚108207562752000 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚262 H@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚665 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 EDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521206 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚41348068 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 5494538240 >5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5264 PDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 5368709120 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚14763228 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚13409078 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚429 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚59395584 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 4517085184 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚3356359 WDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚95939385053696 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚262 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 1019500384768 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 BDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 EDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚12544927 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 845178880 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚108844688801792 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17104896 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚20643840 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚109808675300864 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 1019546516992 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚262 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚41948160 TCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 963927446016 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 4552982528 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚28172304 NGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚68 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚322 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 [JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 961956504814 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚4293 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚108844688801792 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚12544997 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 ^KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚94919884668928 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚108844705906688 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17104896 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚223 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚46132224 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚322 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 122905600 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚13413371 I@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚4876 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚109169460869386 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚28172304 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚14763228 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚13409078 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚109808616247808 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚14763226 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 125829120 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 122905600 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚109169519256814 _runtime‚1072.144392944 SFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚66776064 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 831258624 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚322 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654602240 B9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚5264 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚9188570 WFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 961915156746 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚322 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 4552982528 [HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚95939451829760 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚209 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚28176599 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 5494538240 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 5494538240 K>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59053056 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚108207562752000 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚262 H@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚665 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 EDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521206 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚41348068 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 5494538240 >5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5264 PDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 5368709120 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚14763228 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚13409078 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚429 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚59395584 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 4517085184 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚3356359 WDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚95939385053696 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚262 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 1019500384768 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 BDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 EDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚111504830488064 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17104896 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚110525420568576 IDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚111504771438080 YFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚97384086160384 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚110525437673472 JBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚665 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 122905600 E8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚28591014 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚41945088 UHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚56293376 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 831258624 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚322 F?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚60 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚12742493 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚14997929 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 122905600 `MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚96348476243968 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚322 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 5494538240 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 5494538240 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚41945088 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚28595303 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚3405204 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 IBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚60 KBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚4876 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚20643840 trainer/global_step‚4049 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚9337227 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 4552982528 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚13597372 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 125829120 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 125829120 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚322 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 5368709120 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚262 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚9337229 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 PCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚58387380 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚12742431 IDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4876 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17104896 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4289 ]JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚97384142453760 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚209 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚262 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚223 @7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚5264 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚13597372 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 845178880 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚59395584 KDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚60 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚429 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚13593085 VEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 979350869504 dQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚96348496887808 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚14997931 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚110525437673472 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚4287 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚110848861454592 % _timestamp‚1.7866597410148292e+09 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2FæZÙ¶'œ °O G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚429 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚13593085 VEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 979350869504 dQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚96348496887808 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚14997931 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚110525437673472 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚4287 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚110848861454592 % _timestamp‚1.7866597410148292e+09 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚111504771438080 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚109871566127104 _step‚331 ]LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 977353714868 C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚4289 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 4654602240 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚14997929 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 5368709120 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 5368709120 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 4654602240 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚110525420568576 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚13593085 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚3405264 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚262 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 F>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚111504830488064 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17104896 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚110525420568576 IDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚111504771438080 YFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚97384086160384 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚110525437673472 JBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚665 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 122905600 E8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚28591014 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚41945088 UHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚56293376 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 831258624 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚322 F?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚60 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚12742493 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚14997929 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 122905600 `MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚96348476243968 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚322 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 5494538240 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 5494538240 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚41945088 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚28595303 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚3405204 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 IBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚60 KBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚4876 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚20643840 trainer/global_step‚4049 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚9337227 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 4552982528 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚13597372 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 125829120 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 125829120 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚322 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 5368709120 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚262 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚9337229 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 PCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚58387380 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚12742431 IDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4876 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17104896 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4289 ]JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚97384142453760 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚209 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚262 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚223 @7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚5264 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚13597372 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 845178880 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚59395584 KDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚60Ë1Eº›2 - _wandbruntime‚1089  _runtime‚1089o’Aš»&ž µM bODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚96371544260608 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚28602149 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚4295 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 122905600 [JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 977584287528 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 5494538240 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚15001482 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚4293 ^KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚96371523616768 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚48229376 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 5368709120 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚262 D=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚60 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚13596372 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 5494538240 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚15001482 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17104896 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚209 WDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚97407378367488 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚262 DDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521206 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 5368709120 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚110551760011264 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚28597854 _runtime‚1089.703437822 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚110551760011264 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚110551742906368 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚20643840 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 4517085184 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 5494538240 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚262 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚28597854 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚13596372 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚322 IBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚60 G@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚70 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚59395584 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 1035902980096 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚110875336208168 NADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚58387428 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 831258624 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚110875277820740 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚109897751920640 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚13600665 PDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 125829120 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚322 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚4293 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚111531383897600 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚41948160 SFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚68873216 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 BDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚28602149 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚12745450 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚429 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 122306526 _step‚332 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654602240 WFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 979581938176 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚15001484 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚9339432 % _timestamp‚1.7866597411732218e+09 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚15001484 E6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚322 trainer/global_step‚4049 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 4654602240 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 5368709120 >5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5264 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚111531383897600 NADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚59053056 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 125829120 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚223 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 1035854750720 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 4552982528 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 845178880 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚322 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚41348068 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚9339434 G@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚60 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 K>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59053056 [HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚97407447240704 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17104896 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚111531324844544 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 125829120 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 EDeviceStatsMBž;»%onitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521206 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 5368709120 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚110551760011264 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚28597854 _runtime‚1089.703437822 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚110551760011264 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚110551742906368 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚20643840 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 4517085184 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 5494538240 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚262 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚28597854 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚13596372 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚322 IBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚60 G@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚70 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚59395584 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 1035902980096 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚110875336208168 NADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚58387428 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 831258624 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚110875277820740 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚109897751920640 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚13600665 PDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 125829120 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚322 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚4293 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚111531383897600 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚41948160 SFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚68873216 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 BDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚28602149 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚12745450 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚429 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 122306526 _step‚332 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654602240 WFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 979581938176 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚15001484 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚9339432 % _timestamp‚1.7866597411732218e+09 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚15001484 E6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚322 trainer/global_step‚4049 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 4654602240 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 5368709120 >5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5264 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚111531383897600 NADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚59053056 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 125829120 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚223 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 1035854750720 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 4552982528 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 845178880 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚322 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚41348068 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚9339434 G@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚60 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 K>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59053056 [HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚97407447240704 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17104896 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚111531324844544 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 125829120 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 E8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚262 DDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4876 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17104896 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 122905600 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚112832475093504 IBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚60 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚4287 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚322 @7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚5264 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚41945088 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 4552982528 UHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚70973440 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚262 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚15175581 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚13758187 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚262 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚13762474 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 4654602240 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚12893704 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 5494538240 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚322 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚111841537458176 JBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚665 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚223 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚322 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚9447479 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 4552982528 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚3446225 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚209 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚112832475093504 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚41945088 YFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚98548713875456 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 5494538240 PCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚59049984 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚429 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 1047919321088 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚112169771329380 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚111180855803904 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 5368709120 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚111841554563072 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 trainer/global_step‚4099 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚322 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 122905600 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 831258624 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 M@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚59049984 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚13762474 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 VEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 990937635328 ]LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 990979580416 PIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚78 ]JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚98548784848896 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚50329600 PCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚58387380 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 125829120 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚15175579 IBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚80 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4289 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚262 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 4654602240 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚111841537458176 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚9447477 dQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚97500865527808 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 1047868991488 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 5368709120 C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚4289 F>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 YHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 990937635328 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 _runtime‚1101.631899352 _step‚335 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚111841554563072 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚112832534143488 D;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚5264 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 E8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚28933766 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚15175581 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚13758187 ZIDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 990979580416 KBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚4876 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 122306526 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚4287 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚41348020 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚112832534143488 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚12893624 BDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4876 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17104896 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 122905600 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚112832475093504 IBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚60 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚4287 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚322 @7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚5264 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚41945088 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 4552982528 UHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚70973440 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚262 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚15175581 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚13758187 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚262 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚13762474 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 4654602240 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚12893704 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 5494538240 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚322 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚111841537458176 JBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚665 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚223 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚322 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚9447479 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 4552982528 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚3446225 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚209 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚112832475093504 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚41945088 YFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚98548713875456 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 5494538240 PCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚59049984 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚429 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 1047919321088 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚112169771329380 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚111180855803904 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 5368709120 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚111841554563072 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 trainer/global_step‚4099 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚322 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 122905600 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 831258624 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 M@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚59049984 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚13762474 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 VEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 990937635328 ]LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 990979580416 PIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚78 ]JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚98548784848896 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚50329600 PCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚58387380 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 125829120 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚15175579 IBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚80 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4289 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚262 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 4654602240 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚111841537458176 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚9447477 dQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚97500865527808 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 1047868991488 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 5368709120 C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚4289 F>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 YHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 990937635328 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 _runtime‚1101.631899352 _step‚335 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚111841554563072 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚112832534143488 D;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚5264 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 E8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚28933766 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚15175581 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚13758187 ZIDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 990979580416 KBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚4876 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 122306526 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚4287 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚41348020 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚112832534143488 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚12893624 BDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚262 % _timestamp‚1.7866597532586768e+09 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚111867859795968 SFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚73067520 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 1048113301504 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚28944901 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 845178880 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 BDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚28944901 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 5368709120 DDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚12896634 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚429 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚15179134 PDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 5494538240 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 5494538240 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 5368709120 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚111867876900864 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚9449682 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 831258624 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚4295 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚13761474 NADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚59053056 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 4517085184 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚28940606 K>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59053056 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 4654602240 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚112859087553024 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚28940606 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚52423680 G@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚60 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚322 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚15179132 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚112196129308148 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚209 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521206 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 122306526 D=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚60 XGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 991210652160 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚3447029 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚20643840 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚59395584 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚41348068 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 125829120 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚4293 I@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚4876 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚322 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 122905600 H@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚665 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚13765767 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚322 _step‚336 EDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654602240 WFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 991168704000 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 5368709120 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚13761474 >5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5264 ^KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚97523892256768 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 TCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 991168704000 [JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 991210652160 [HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚98572078625792 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 1048165725184 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚112196187695576ˆ@+‚À&­ ºM C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚4295 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚13761474 NADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚59053056 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 4517085184 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚28940606 K>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59053056 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 4654602240 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚112859087553024 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚28940606 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚52423680 G@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚60 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚322 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚15179132 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚112196129308148 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚209 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521206 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 122306526 D=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚60 XGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 991210652160 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚3447029 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚20643840 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚59395584 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚41348068 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 125829120 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚4293 I@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚4876 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚322 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 122905600 H@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚665 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚13765767 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚322 _step‚336 EDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654602240 WFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 991168704000 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 5368709120 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚13761474 >5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5264 ^KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚97523892256768 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 TCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 991168704000 [JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 991210652160 [HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚98572078625792 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 1048165725184 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚112196187695576 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚111207041597440 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚322 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚262 % _timestamp‚1.7866597532586768e+09 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚111867859795968 SFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚73067520 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 1048113301504 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚28944901 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 845178880 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 BDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚28944901 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 5368709120 DDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚12896634 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚429 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚15179134 PDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 5494538240 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 5494538240 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 5368709120 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚111867876900864 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚9449682 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 831258624 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0ЊƒSË2® - _wandbruntime‚1101  _runtime‚1101P¶ r¯ ì % _timestamp‚1.7866597532596076e+09 _runtime‚1101.790203707 _step‚337 0loss/FRt1/assignment_loss‚0.8194742901531545 0loss/FRt2/assignment_loss‚0.8479362251490759 %loss/total_loss‚2.722930859541636 epoch‚10 trainer/global_step‚4099 /loss/FRt1/detection_loss‚0.5411977388627406 /loss/FRt2/detection_loss‚0.5143226053766652 øqÌw° ñ trainer/global_step‚4099 /loss/FRt1/detection_loss‚0.5411977388627406 /loss/FRt2/detection_loss‚0.5143226053766652 % _timestamp‚1.7866597532596076e+09 _runtime‚1101.790203707 _step‚337 0loss/FRt1/assignment_loss‚0.8194742901531545 0loss/FRt2/assignment_loss‚0.8479362251490759 %loss/total_loss‚2.722930859541636 epoch‚10ÑDú•± ‚(:Š ±‡ùÓ”ýá gpu.0.gpu‚48.0 gpu.0.memory‚8, gpu.0.memoryAllocated‚7.0313262939453125) gpu.0.memoryAllocatedBytes‚ 6039863296 gpu.0.temp‚37.0 gpu.0.powerWatts‚197.796' gpu.0.enforcedPowerLimitWatts‚500.0 gpu.0.powerPercent‚39.5592 gpu.0.smClock‚1410 gpu.0.memoryClock‚1593$ gpu.0.correctedMemoryErrors‚1955# gpu.0.uncorrectedMemoryErrors‚0é–:ˆ² ‚(:ý ±‡ùÓ÷´Òâ disk.sda2.in‚0& disk.nvme0n1.out‚17096.96337890625 disk.dm-1.out‚22.40625# disk.dm-0.out‚17096.96337890625 disk.sda3.out‚22.40625" proc.memory.rssMB‚ 2384.3046875 cpu‚26.912576575237278 proc.cpu.threads‚17 disk.sda.in‚ 10591.078125 disk.sda2.out‚0 disk.nvme0n1.in‚ 352.71484375$ memory_percent‚65.87104797363281 network.sent‚ 5.532944e+06 network.recv‚ 2.292365e+06 disk.sda.out‚22.40625 disk.sda1.in‚0 disk.dm-1.in‚ 10591.078125 disk.drbd2165.out‚0( disk.nvme0c0n1.out‚17096.96337890625% proc.memory.availableMB‚ 5591.6875) disk./.usagePercent‚30.69930398514845$ disk./.usageGB‚457.4418144226074 disk.sda1.out‚0 disk.drbd2165.in‚0 disk.dm-0.in‚ 352.71484375 disk.sda3.in‚ 10591.078125" disk.nvme0c0n1.in‚ 352.71484375* proc.memory.percent‚14.552640914916992 ¡2³ - _wandbruntime‚1113  _runtime‚1113œ‰W`Ç´ Á trainer/global_step‚4149 % _timestamp‚1.7866597650926247e+09 _runtime‚1113.624931517 _step‚338 & lr-AdamW/pg1‚0.0013686161976899254 & lr-AdamW/pg2‚0.0013686161976899254S¥p̵ Æ & lr-AdamW/pg1‚0.0013686161976899254 & lr-AdamW/pg2‚0.0013686161976899254 trainer/global_step‚4149 % _timestamp‚1.7866597650926247e+09 _runtime‚1113.624931517 _step‚338Ò–— P2¶ - _wandbruntime‚1113  _runtime‚1113OÆÁ·'· ±O RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 122905600 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 1002566346240 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚209 BDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚114160178748928 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 1002524401152 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚9557729 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚3486924 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 1060167645184 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚112490145480704 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚262 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 1002524401152 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚113157654347776 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚13044653 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 5494538240 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚15353231 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17104896 IDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4876 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚29280807 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 122306526 trainer/global_step‚4149 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚9557727 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚44038144 @7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚5264 `MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚98653213523968 E8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚29276518 IDeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4289 D;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚5264 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚59395584 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 5368709120 F>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665}¿T¸ ¶O IDeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4289 D;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚5264 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚59395584 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 5368709120 F>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 122905600 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 1002566346240 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚209 BDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚114160178748928 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 1002524401152 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚9557729 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚3486924 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 1060167645184 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚112490145480704 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚262 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 1002524401152 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚113157654347776 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚13044653 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 5494538240 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚15353231 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17104896 IDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4876 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚29280807 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 122306526 trainer/global_step‚4149 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚9557727 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚44038144 @7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚5264 `MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚98653213523968 E8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚29276518Ós;‡ë2¹ - _wandbruntime‚1113  _runtime‚1113B6¬ÉÀ&º ºM S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚113516980795556 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 122905600 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521206 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 D=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚60 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚15356784 I@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚4876 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 1060418243584 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚112516314234880 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 4517085184 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚4295 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 1002755469824 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚13047594 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 4552982528 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚29283358 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚13926576 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 1060365819904 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 125829120 IBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚60 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚13930869 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚3487736 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 125829120 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654602240 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 1002797417984 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 122905600 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚15356784 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 G@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚60 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚322 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚429 EDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59053056 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚59395584 NADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚58387428 _step‚340 D6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚322 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚41948160 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚113183993790464 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17104896 B9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚5264 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚20643840 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚113183993790464 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚13047670 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 5494538240 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚262 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 125829120 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚29287653 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚9559932 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 5494538240 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 5368709120 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 1002755469824 G@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚76 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 122306526 PDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 $ _timestamp‚1.786659765253513e+09 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 H@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚665 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚4293 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚15356782 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚3487662 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚209 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚52423680 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚112516331274240 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚113183976685568 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 >5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5264 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚4293 ^KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚98676260896768 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚113517039182984 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 5494538240 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 1000707908744 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚41348068 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 5368709120 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚41948160 SFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚73067520 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 B6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚322 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 1002797417984 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1sÙ®;Å&» ¿M TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 4552982528 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 5368709120 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 $ _timestamp‚1.786659765253513e+09 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 H@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚665 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚4293 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚15356782 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚3487662 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚209 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚52423680 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚112516331274240 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚113183976685568 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 >5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5264 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚4293 ^KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚98676260896768 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚113517039182984 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 5494538240 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 1000707908744 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚41348068 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 5368709120 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚41948160 SFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚73067520 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 B6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚322 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 1002797417984 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚113516980795556 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 122905600 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521206 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 D=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚60 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚15356784 I@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚4876 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 1060418243584 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚112516314234880 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 4517085184 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚4295 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 1002755469824 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚13047594 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 4552982528 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚29283358 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚13926576 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 1060365819904 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 125829120 IBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚60 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚13930869 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚3487736 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 125829120 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654602240 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 1002797417984 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 122905600 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚15356784 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 G@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚60 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚322 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚429 EDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59053056 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚59395584 NADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚58387428 _step‚340 D6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚322 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚41948160 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚113183993790464 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17104896 B9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚5264 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚20643840 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚113183993790464 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚13047670 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 5494538240 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚262 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 125829120 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚29287653 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚9559932 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 5494538240 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 5368709120 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 1002755469824 G@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚76 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 122306526 PDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚114473771237376 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 4552982528 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 1014153112064 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 1014111166976 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 125829120 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚322 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 KBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚4876 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚114811474304196 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚262 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 4654602240 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚114473788342272 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 M@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚59049984 PCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚58387380 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚114811415916816 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚115487941454336 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚100878040996352 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚15530881 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17104896 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4289 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 1072438188544 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚429 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚322 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚115487882404352 IBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚60 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚9667979 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚14092678 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 4654602240 PIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚67 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚59395584 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 trainer/global_step‚4199 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚115487882404352 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 1014153112064 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚209 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 5368709120 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚29623559 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚41348020 G>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4876 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17104896 D;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚5264 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 _runtime‚ 1126.2059124 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚115487941454336 E8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚29619270 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 1014111166976 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚41945088 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 1012039146692 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 125829120 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚262 _step‚343 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 5368709120 @7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚5264 IDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4876 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17104896 D;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚5264 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 _runtime‚ 1126.2059124 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚115487941454336 E8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚29619270 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 1014111166976 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚41945088 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 1012039146692 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 125829120 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚262 _step‚343 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 5368709120 @7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚5264 IDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚114473771237376 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 4552982528 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 1014153112064 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 1014111166976 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 125829120 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚322 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 KBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚4876 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚114811474304196 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚262 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 4654602240 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚114473788342272 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 M@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚59049984 PCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚58387380 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚114811415916816 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚115487941454336 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚100878040996352 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚15530881 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17104896 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4289 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 1072438188544 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚429 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚322 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚115487882404352 IBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚60 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚9667979 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚14092678 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 4654602240 PIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚67 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚59395584 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 trainer/global_step‚4199 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚115487882404352 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 1014153112064×ÄT\©2Ç - _wandbruntime‚1126  _runtime‚1126Ó7Ã&È ½M WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚114500093575168 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚52423680 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 5494538240 _step‚344 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 1012228371284 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 125829120 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚14095971 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚115514435810816 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 4552982528 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚115514494863872 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17104896 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚114500110680064 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚41348068 G@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚60 G@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚75 SFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚73067520 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚113825620951040 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 EDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59053056 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 1014342235648 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚13199049 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚13198974 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚322 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 NADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚58387428 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚115514494863872 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚15534434 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚14095971 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 1012269719352 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17104896 EDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521206 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚113825603911680 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 4517085184 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 5368709120 trainer/global_step‚4199 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚29626110 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 122905600 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚14091678 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚9670182 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 831258624 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚114837832282964 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚262 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 1014342235648 I@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚4876 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 125829120 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚429 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚9670184 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 4654602240 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚3528865 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚4293 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 1014384183808 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚29626110 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚322 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚14091678 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚41948160 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚114837890670392 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 122306526 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 5494538240 % _timestamp‚1.7866597778316047e+09 PDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654602240 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 4552982528 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚41948160 bODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚99828650180608 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚322 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚29630405 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚15534432 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚20643840 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚114500110680064 B9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚5264 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 1072682855936 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 IBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚60 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 D=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚60 NGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚73 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚4295 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚100901333036544 B5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5264 DDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 125829120 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚100901259969024 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 1072630432256 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0£È,Óà É ÂM A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚4295 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚100901333036544 B5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5264 DDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 125829120 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚100901259969024 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 1072630432256 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚114500093575168 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚52423680 =7DeviceStatsMonitor.onLñoâ_train_batch_end/num_alloc_retries‚0 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 5494538240 _step‚344 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 1012228371284 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 125829120 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚14095971 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚115514435810816 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 4552982528 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚115514494863872 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17104896 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚114500110680064 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚41348068 G@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚60 G@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚75 SFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚73067520 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚113825620951040 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 EDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59053056 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 1014342235648 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚13199049 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚13198974 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚322 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 NADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚58387428 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚115514494863872 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚15534434 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚14095971 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 1012269719352 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17104896 EDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521206 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚113825603911680 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 4517085184 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 5368709120 trainer/global_step‚4199 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚29626110 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 122905600 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚14091678 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚9670182 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 831258624 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚114837832282964 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚262 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 1014342235648 I@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚4876 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 125829120 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚429 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚9670184 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 4654602240 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚3528865 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚4293 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 1014384183808 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚29626110 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚322 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚14091678 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚41948160 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚114837890670392 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 122306526 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 5494538240 % _timestamp‚1.7866597778316047e+09 PDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654602240 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 4552982528 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚41948160 bODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚99828650180608 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚322 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚29630405 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚15534432 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚20643840 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚114500110680064 B9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚5264 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 1072682855936 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 IBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚60 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 D=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚60 NGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚73 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0ØF ×2Ê - _wandbruntime‚1126  _runtime‚1126'%„pË ê _step‚345 0loss/FRt1/assignment_loss‚0.8561727628942539 0loss/FRt2/assignment_loss‚0.7479401878378699 /loss/FRt1/detection_loss‚0.5592251531260952 $loss/total_loss‚2.65502796306568 trainer/global_step‚4199 % _timestamp‚1.7866597778326054e+09 /loss/FRt2/detection_loss‚0.4916898592074613 epoch‚10 _runtime‚ 1126.36244888íY“>uÌ ï _step‚345 0loss/FRt1/assignment_loss‚0.8561727628942539 0loss/FRt2/assignment_loss‚0.7479401878378699 /loss/FRt1/detection_loss‚0.5592251531260952 $loss/total_loss‚2.65502796306568 trainer/global_step‚4199 % _timestamp‚1.7866597778326054e+09 /loss/FRt2/detection_loss‚0.4916898592074613 epoch‚10 _runtime‚ 1126.36244888Ùµûµ2Í - _wandbruntime‚1138  _runtime‚1138$¸ ÇÎ Á & lr-AdamW/pg1‚0.0013617402082556746 & lr-AdamW/pg2‚0.0013617402082556746 trainer/global_step‚4249 % _timestamp‚1.7866597903107069e+09 _runtime‚1138.840876631 _step‚346]ÍÆàÌÏ Æ % _timestamp‚1.7866597903107069e+09 _runtime‚1138.840876631 _step‚346 & lr-AdamW/pg1‚0.0013617402082556746 & lr-AdamW/pg2‚0.0013617402082556746 trainer/global_step‚4249ÚýS2Ð - _wandbruntime‚1138  _runtime‚1138“äý»'Ñ µO SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 845178880 _runtime‚1138.845953547 E8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚29962022 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚223 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 831258624 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚322 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 M@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚59049984 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚100957971447808 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚20643840 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚41945088 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚41348020 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚15708531 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚15708531 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚116815586059776 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚29966311 D;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚5264 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚3568837 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17104896 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4289 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 1084671828992 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 122306526 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 5494538240 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 1025739877888 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚102042622632960 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚115789905231872 KBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚4876 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚9778227 PIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚62 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚209 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚41940992 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 1023559609280 _step‚347 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 5494538240 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 5494538240 IBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚60 C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚4289 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 4552982528 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚116815645109760 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚4287 PCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚58387380 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚262 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 1025697932800 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚100957950803968 BDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚116815645109760 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 122905600 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚102042685217792 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 5368709120 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚3568775 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 KDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚60 % _timestamp‚1.7866597903117793e+09 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚429 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 125829120 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚262 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚59395584 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 5368709120 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 125829120 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚322 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 1025697932800 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚9778229 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚4287 @7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚5264 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚14253493 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚14257780 trainer/global_step‚4249 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚115789888126976 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚13347002 G>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4876 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 4552982528 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚15708529 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚14253493 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚116815586059776 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 1084713769984 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 122905600 IBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚64 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚14257780 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚41945088 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17104896 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚116132267404224 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 4628521206 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 PCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚59049984 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 4654602240 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚115108724834304 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚322 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚15708529 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 4654602240 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 5368709120 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚115789888126976 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚322 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 125829120 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚2626§ã”À'Ò ºO KBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚4876 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚9778227 PIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚62 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚209 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚41940992 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 1023559609280 _step‚347 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 5494538240 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 5494538240 IBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚60 C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚4289 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 4552982528 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚116815645109760 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚4287 PCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚58387380 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚262 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 1025697932800 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚100957950803968 BDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚116815645109760 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 122905600 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚102042685217792 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 5368709120 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚3568775 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 KDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚60 % _timestamp‚1.7866597903117793e+09 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚429 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 125829120 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚262 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚59395584 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 5368709120 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 125829120 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚322 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 1025697932800 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚9778229 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚4287 @7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚5264 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚14253493 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚14257780 trainer/global_step‚4249 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚115789888126976 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚13347002 G>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4876 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 4552982528 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚15708529 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚14253493 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚116815586059776 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 1084713769984 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 122905600 IBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚64 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚14257780 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚41945088 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17104896 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚116132267404224 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 4628521206 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 PCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚59049984 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 4654602240 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚115108724834304 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚322 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚15708529 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 4654602240 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 5368709120 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚115789888126976 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚322 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 125829120 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚262 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 845178880 _runtime‚1138.845953547 E8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚29962022 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚223 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 831258624 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚322 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 M@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚59049984 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚100957971447808 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚20643840 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚41945088 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚41348020 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚15708531 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚15708531 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚116815586059776 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚29966311 D;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚5264 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚3568837 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17104896 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4289 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 1084671828992 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 122306526 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 5494538240 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 1025739877888 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚102042622632960 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚115789905231872ÛYáñ 2Ó - _wandbruntime‚1139  _runtime‚1139>vì Ô ¿M NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚15712084 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚14261073 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚116842198519296 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚115816227569664 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚14261073 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚29973157 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 1025929001472 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 5368709120 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚41348068 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 1025970949632 H@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚665 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 125829120 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚262 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚115816227569664 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 1025929001472 SFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚62581760 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 1023790181892 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 122905600 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 1023831529960 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚4295 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚14256780 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚115816210464768 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚322 EDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521206 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚115134893588480 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 5494538240 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚29968862 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚4293 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 4552982528 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚3569606 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_poÉÆ]Ùol.allocated‚115134910627840 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 5494538240 B9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚5264 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 125829120 DDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59053056 NADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚59053056 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚115816210464768 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚14256780 _runtime‚1139.009344341 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚15712082 I@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚4876 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 5368709120 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 4654602240 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚13350038 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 831258624 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚15712084 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚209 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚41937920 IBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚60 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚4293 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚223 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚102065913791488 _step‚348 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚41948160 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚29968862 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚9780432 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 G@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚60 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 4552982528 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 1084957552640 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 5494538240 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚9780434 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚100980998176768 B5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5264 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚429 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚262 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 1025970949632 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚262 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚13350109 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚102065976373248 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 125829120 % _timestamp‚1.7866597904789681e+09 E6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚322 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚116842198519296 PDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚29973157 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚116158742157800 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚20643840 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 NADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚58387428 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚116158683770372 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚15712082 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚116842139466240 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654602240 NGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚69 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚100981018820608 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚59395584 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17104896 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚41948160 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17104896 G@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚71 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 D=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚60ª4“ Ê&Õ ÄM K>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59053056 NADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚59053056 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚115816210464768 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚14256780 _runtime‚1139.009344341 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚15712082 I@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚4876 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 5368709120 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 4654602240 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚13350038 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 831258624 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚15712084 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚209 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚41937920 IBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚60 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚4293 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚223 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚102065913791488 _step‚348 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚41948160 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚29968862 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚9780432 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 G@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚60 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 4552982528 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 1084957552640 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 5494538240 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚9780434 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚100980998176768 B5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5264 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚429 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚262 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 1025970949632 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚262 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚13350109 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚102065976373248 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 125829120 % _timestamp‚1.7866597904789681e+09 E6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚322 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚116842198519296 PDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚29973157 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚116158742157800 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚20643840 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 NADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚58387428 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚116158683770372 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚15712082 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚116842139466240 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654602240 NGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚69 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚100981018820608 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚59395584 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17104896 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚41948160 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17104896 G@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚71 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 D=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚60 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚15712084 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚14261073 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚116842198519296 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚115816227569664 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚14261073 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚29973157 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 1025929001472 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 5368709120 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚41348068 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 1025970949632 H@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚665 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 125829120 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚262 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚115816227569664 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 1025929001472 SFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚62581760 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 1023790181892 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 122905600 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 1023831529960 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚4295 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚14256780 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚115816210464768 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚322 EDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521206 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚115134893588480 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 5494538240 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚29968862 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚4293 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 4552982528 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚3569606 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚115134910627840 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 5494538240 B9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚5264 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 125829120 DDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4876 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚322 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚14418595 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 1037326643712 UHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚68876288 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚103207240181760 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 _runtime‚1151.357933829 @7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚5264 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚4287 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 4654602240 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚41945088 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚322 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚262 % _timestamp‚1.7866598028242884e+09 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚15886179 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 1037284698624 IDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚118143289715200 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚118143289715200 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 1096968970240 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 125829120 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚3609764 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚20643840 PCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚58387380 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 4517085184 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 125829120 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚15886181 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 4654602240 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚117106005016576 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 IDeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4289 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 1035121419888 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 5368709120 JBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚665 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚4287 IBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚60 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚118143348765184 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚223 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 4628521206 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚15886179 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 831258624 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 5494538240 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚117453177279012 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 125829120 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚262 PCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚59049984 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚13498243 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 1037284698624 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚14422882 KBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚4876 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 1096920737792 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 5494538240 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17104896 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚103207309058048 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚102110319443968 F?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚60 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 F>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚14422882 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚116417997471744 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚117106022121472 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚118143348765184 IBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚77 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚117453118891632 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚30309063 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚48232448 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚59395584 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚117106005016576 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚9888477 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚3609689 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚209 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 5368709120 KDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚60 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 4552982528 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚41945088 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 122905600 D;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚5264 PIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚75 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0ßRà ºO E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚41348020 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚118143289715200 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚118143289715200 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 1096968970240 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 125829120 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚3609764 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚20643840 PCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚58387380 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 4517085184 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 125829120 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚15886181 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 4654602240 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚117106005016576 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 IDeviceStatsMonitor.on_train_batch_start/allocation.all.current‚vxm§4289 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 1035121419888 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 5368709120 JBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚665 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚4287 IBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚60 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚118143348765184 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚223 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 4628521206 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚15886179 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 831258624 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 5494538240 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚117453177279012 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 125829120 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚262 PCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚59049984 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚13498243 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 1037284698624 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚14422882 KBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚4876 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 1096920737792 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 5494538240 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17104896 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚103207309058048 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚102110319443968 F?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚60 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 F>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚14422882 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚116417997471744 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚117106022121472 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚118143348765184 IBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚77 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚117453118891632 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚30309063 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚48232448 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚59395584 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚117106005016576 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚9888477 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚3609689 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚209 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 5368709120 KDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚60 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 4552982528 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚41945088 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 122905600 D;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚5264 PIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚75 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 G>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4876 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚322 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚14418595 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 1037326643712 UHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚68876288 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚103207240181760 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 _runtime‚1151.357933829 @7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚5264 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚4287 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 4654602240 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚41945088 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚322 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚262 % _timestamp‚1.7866598028242884e+09 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚15886179 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 1037284698624 IDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚30315909 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚15889732 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚103230533437440 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 845178880 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚48229376 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 4517085184 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 4552982528 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚20643840 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521206 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 5368709120 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 125829120 _step‚352 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17104896 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚117132344459264 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚41348068 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚262 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚14426175 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚9890682 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚3610543 IBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚60 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚14426175 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚3610478 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 1097166620672 EDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59053056 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 4654602240 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 1037515767296 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚118169902174720 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚322 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 5368709120 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚15889732 % _timestamp‚1.7866598029870968e+09 NADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚59053056 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚15889734 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚103230602310656 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚59395584 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 B6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚322 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚116444200304640 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚117479535257780 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚322 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 122905600 SFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚68873216 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 831258624 trainer/global_step‚4299 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 5494538240 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 125829120 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚117132327354368 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 4552982528 E8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚14421882 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 1037557715456 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 1035351992500 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚322 >5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5264 NGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚65 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚4293 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚9890684 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚117479593645208 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚116444183265280 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚4295 PDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654602240 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚30315909 DDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚4293 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 G@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚60 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚41948160 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚13501160¨_“eÊ&ã ÄM K>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59053056 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 4654602240 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 1037515767296 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚118169902174720 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚322 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 5368709120 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚15889732 % _timestamp‚1.7866598029870968e+09 NADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚59053056 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚15889734 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚103230602310656 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚59395584 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 B6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚322 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚116444200304640 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚117479535257780 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚322 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 122905600 SFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚68873216 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 831258624 trainer/global_step‚4299 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 5494538240 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 125829120 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚117132327354368 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 4552982528 E8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚14421882 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 1037557715456 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 1035351992500 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚322 >5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5264 NGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚65 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚4293 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚9890684 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚117479593645208 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚116444183265280 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚4295 PDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654602240 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚30315909 DDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚4293 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 G@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚60 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚41948160 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚13501160 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚102133387460608 D=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚60 _runtime‚1151.517432038 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 122306526 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚117132344459264 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚13501227 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 122905600 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚14421882 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 1035393340568 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚15889734 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17104896 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚30315909 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚15889732 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚103230533437440 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 845178880 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚48229376 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 4517085184 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 4552982528 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚20643840 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521206 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 5368709120 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 125829120 _step‚352 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17104896 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚117132344459264 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚41348068 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚262 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚14426175 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚9890682 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚3610543 IBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚60 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚14426175 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚3610478 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 1097166620672 EDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚119470993370624 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚41945088 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚20643840 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚322 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 1048871464448 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 5368709120 trainer/global_step‚4349 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 4552982528 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17104896 IDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 4654602240 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚14583697 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 UHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚58390528 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 1109240851968 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 125829120 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚118422121906176 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚16063831 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚13649494 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚16063831 JBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚665 KBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚4876 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 831258624 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 1048871464448 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚16063829 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 5494538240 TFDeviceStatsMonitor.on_train_batch_W{˜½start/reserved_bytes.small_pool.peak‚ 125829120 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚223 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚209 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚322 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚118774028766420 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 5368709120 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚4287 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 122905600 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚104371949579776 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17104896 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 4552982528 IBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚65 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚41945088 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚3650765 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 BDeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4289 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚104371891189248 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 5494538240 PIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚63 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 1109203105280 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 @7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚5264 G>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4876 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 1048913409536 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚3650702 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 845178880EFVñ¿'î ¹O MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 125829120 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚14587984 PCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚59049984 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚429 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 1048913409536 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚14583697 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 1046683230496 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚103262688083968 PCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚58387380 IBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚60 _step‚355 IDeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4289 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚104371891189248 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 5494538240 PIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚63 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 1109203105280 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 @7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚5264 G>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4876 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 1048913409536 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚3650702 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 845178880 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚59395584 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚16063829 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚262 F>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚119470993370624 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚41945088 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚20643840 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚322 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 1048871464448 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 5368709120 trainer/global_step‚4349 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 4552982528 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17104896 IDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 4654602240 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚14583697 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 UHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚58390528 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 1109240851968 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 125829120 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚118422121906176 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚16063831 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚13649494 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚16063831 JBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚665 KBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚4876 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 831258624 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 1048871464448 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚16063829 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 5494538240 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 125829120 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚223 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚209 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚322 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚118774028766420 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 5368709120 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚4287 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 122905600 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚104371949579776 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17104896 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 4552982528 IBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚65 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚41945088 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚3650765 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 BDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59053056 NADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚58387428 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 1046955151176 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚30654366 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚3651479 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚118800445132616 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521206 PDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654602240 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚41948160 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚10000934 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚10000932 NGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚74 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚4295 G@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚76 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 125829120 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 122905600 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 _runtime‚1164.332833259 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 125829120 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚41348068 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 5368709120 trainer/global_step‚4349 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17104896 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚119497546777088 B9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚5264 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 4517085184 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚262 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚103285756100608 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 122306526 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚262 D=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚60 % _timestamp‚1.7866598158024564e+09 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚429 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 G@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚60 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 122905600 H@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚665 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚16067382 E6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚322 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚119497605830144 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 4552982528 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚4293 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚41948160 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚13652411 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚117753489981440 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚262 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚322 D6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚322 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 5494538240 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 1049102533120 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 1109444269568 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚59395584 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚14586984 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚16067384 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚4293 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 >5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5264 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚104395248599552 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚30654366 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚16067382 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 831258624 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚48229376 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 5494538240 IBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚60 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚118448461348864 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 1046913803108 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 1049144481280 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚13652487 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚104395179726336 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 845178880 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚119497605830144 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 1049102533120 EDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 _step‚356 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚118448461348864 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚30658661 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚223 SFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚68873216 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚20643840 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚118448444243968Ný†Âhñ ÆM >5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5264 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚104395248599552 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚30654366 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚16067382 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 831258624 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚48229376 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 5494538240 IBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚60 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚118448461348864 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 1046913803108 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 1049144481280 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚13652487 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚104395179726336 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 845178880 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚119497605830144 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 1049102533120 EDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 _step‚356 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚118448461348864 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚30658661 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚223 SFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚68873216 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚20643840 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚118448444243968 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚103285735456768 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 1109492498944 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚118448444243968 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚14586984 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚209 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚118800386745188 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚14591277 K>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59053056 NADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚58387428 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 1046955151176 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚30654366 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚3651479 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚118800445132616 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521206 PDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654602240 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚41948160 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚10000934 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚10000932 NGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚74 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚4295 G@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚76 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 125829120 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 122905600 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 _runtime‚1164.332833259 WIDeviceStatsMonitor.on_train_batch_end/reservedòÕÍd_bytes.small_pool.allocated‚ 125829120 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚41348068 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 5368709120 trainer/global_step‚4349 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17104896 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚119497546777088 B9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚5264 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 4517085184 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚262 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚103285756100608 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 122306526 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚262 D=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚60 % _timestamp‚1.7866598158024564e+09 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚429 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 G@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚60 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 122905600 H@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚665 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚16067382 E6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚322 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚119497605830144 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 4552982528 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚4293 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚41948160 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚13652411 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚117753489981440 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚262 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚322 D6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚322 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 5494538240 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 1049102533120 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 1109444269568 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚59395584 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚14586984 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚16067384 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚4293 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0äyV2ò - _wandbruntime‚1164  _runtime‚1164åasó í &loss/total_loss‚2.6140185535146983 _runtime‚1164.333408776 epoch‚10 trainer/global_step‚4349 % _timestamp‚1.7866598158035514e+09 _step‚357 0loss/FRt1/assignment_loss‚0.8197590952362517 0loss/FRt2/assignment_loss‚0.7914477292729895 /loss/FRt1/detection_loss‚0.5378724173575579 /loss/FRt2/detection_loss‚0.4649393116478991q±Oxô ò 0loss/FRt1/assignment_loss‚0.8197590952362517 0loss/FRt2/assignment_loss‚0.7914477292729895 /loss/FRt1/detection_loss‚0.5378724173575579 /loss/FRt2/detection_loss‚0.4649393116478991 &loss/total_loss‚2.6140185535146983 _runtime‚1164.333408776 epoch‚10 trainer/global_step‚4349 % _timestamp‚1.7866598158035514e+09 _step‚357åHzNb–õ ‚(:‹ í‡ùӥǖá gpu.0.gpu‚58.0 gpu.0.memory‚10, gpu.0.memoryAllocated‚7.0313262939453125) gpu.0.memoryAllocatedBytes‚ 6039863296 gpu.0.temp‚36.0 gpu.0.powerWatts‚227.869' gpu.0.enforcedPowerLimitWatts‚500.0 gpu.0.powerPercent‚45.5738 gpu.0.smClock‚1410 gpu.0.memoryClock‚1593$ gpu.0.correctedMemoryErrors‚1955# gpu.0.uncorrectedMemoryErrors‚0QžœÜö ‚(:„ í‡ùÓáæ±ã" proc.memory.rssMB‚ 2384.3046875* disk./.usagePercent‚30.881496611033704 disk.sda2.in‚0$ disk.nvme0n1.out‚22970.310546875 disk.drbd2165.out‚0$ memory_percent‚66.25049114227295) proc.memory.availableMB‚ 5529.51953125 proc.cpu.threads‚17 disk.sda2.out‚0 network.sent‚ 5.84884e+06$ disk./.usageGB‚460.1566162109375 disk.sda3.out‚ 23.66015625" disk.nvme0c0n1.in‚ 358.51171875& disk.nvme0c0n1.out‚22970.310546875 disk.dm-1.in‚ 10591.078125* proc.memory.percent‚14.552640914916992 disk.sda3.in‚ 10591.078125 disk.sda.in‚ 10591.078125 disk.sda.out‚ 23.66015625 disk.nvme0n1.in‚ 358.51171875 disk.dm-1.out‚ 23.66015625 cpu‚26.921443853465153 network.recv‚ 2.418429e+06 disk.sda1.in‚0 disk.sda1.out‚0 disk.drbd2165.in‚0 disk.dm-0.in‚ 358.51171875! disk.dm-0.out‚22969.810546875²NÙ92÷ - _wandbruntime‚1177  _runtime‚1177#†¸PÇø Á & lr-AdamW/pg2‚0.0013511321099416882 trainer/global_step‚4399 % _timestamp‚1.7866598285842702e+09 _runtime‚1177.114312899 _step‚358 & lr-AdamW/pg1‚0.0013511321099416882ézFZÌù Æ _step‚358 & lr-AdamW/pg1‚0.0013511321099416882 & lr-AdamW/pg2‚0.0013511321099416882 trainer/global_step‚4399 % _timestamp‚1.7866598285842702e+09 _runtime‚1177.114312899æ0º•G2ú - _wandbruntime‚1177  _runtime‚1177 SG ½'û ·O TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17104896 PCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚59049984 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚262 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚14748799 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 125829120 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 1060500175360 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚13800393 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚119036576825344 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 5494538240 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 4654602240 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 4552982528 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚104415056723968 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 1121516482560 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 5494538240 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚322 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 5368709120 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚322 KDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚60 IBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚60 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 trainer/global_step‚4399 _runtime‚1177.118915039 E8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚30990278 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚119738255900672 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚13800459 F>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚59395584 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚120094880253828 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 5494538240 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17104896 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚3691416 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚104415077367808 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 IDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4876 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 4552982528 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚105536593850368 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 845178880 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 5368709120 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚14753086 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 122905600 IBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚66 PIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚64 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚120094821866448 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚322 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 1060500175360 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 1058286389124 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚41348020 M@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚59049984 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚20643840 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 F?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚60 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚16241481 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚41945088 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 1060458230272 PCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚58387380 _step‚359 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚223 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚105536522876928 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚14748799 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚10108977 UHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚70973440 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 4517085184 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚14753086 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 4628521206 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 5368709120 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚120798697026048 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚4287 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 831258624 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚119738238795776 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚120798697026048 IDeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4289 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 1121466152960 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 1058245041104 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚120798756076032%aú¨Â'ü ¼O YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚120798756076032 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17104896 PCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚59049984 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚262 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚14748799 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 125829120 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 1060500175360 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚13800393 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚119036576825344 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 5494538240 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 4654602240 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 4552982528 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚104415056723968 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 1121516482560 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 5494538240 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚322 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 5368709120 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚322 KDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚60 IBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚60 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 trainer/global_step‚4399 _runtime‚1177.118915039 E8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚30990278 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚119738255900672 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚13800459 F>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚59395584 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚120094880253828 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 5494538240 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17104896 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚3691416 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚104415077367808 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 IDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4876 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 4552982528 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚105536593850368 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 845178880 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 5368709120 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚14753086 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 122905600 IBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚66 PIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚64 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚120094821866448 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚322 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 1060500175360 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 1058286389124 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚41348020 M@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚59049984 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚20643840 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 F?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚60 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚16241481 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚41945088 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 1060458230272 PCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚58387380 _step‚359 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚223 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚105536522876928 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚14748799 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚10108977 UHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚70973440 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 4517085184 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚14753086 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 4628521206 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 5368709120 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚120798697026048 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚4287 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 831258624 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚119738238795776 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚120798697026048 IDeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4289 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 1121466152960 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 1058245041104 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0çË{ÿ]2ý - _wandbruntime‚1177  _runtime‚1177ÇšÐWPþ ¿M PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 122905600 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚14756379 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 D=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚60 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚104438124740608 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚120121238232596 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521206 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 1058475613716 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 trainer/global_step‚4399 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚120825250432512 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚31001413 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚16245034 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 4552982528 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚3692221 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚41348068 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚4293 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚119764561133568 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 125829120 D8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 NADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚58387428 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚10111182 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚14752086 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 1060731247104 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚13803465 NGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚60 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 5494538240 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚16245032 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚16245034 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚16245032 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚322 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 5494538240 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 PDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654602240 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 831258624 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 1121714853888 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚59395584 G@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚60 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚41948160 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚322 EDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59053056 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚31001413 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚13803403 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 125829120 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚209 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚262 # _timestamp‚1.78665982875289e+09 EDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚262 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 1060731247104 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚120825309485568 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚4293 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 1121760986112 _runtime‚1177.283257667 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 5368709120 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 1058516961784 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚262 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 5494538240 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 5368709120 G@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚62 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚3692281 SFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚66776064 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚105559818950656 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚4295 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚41948160 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚10111184 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 122306526 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚322 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚119764561133568 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚30997118 H@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚665 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚14752086 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 4654602240 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚119764578238464 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚104438104096768 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 125829120 IBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚60 >5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5264 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚14756379 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17104896 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 B8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 NADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚58387428 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚10111182 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚14752086 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 1060731247104 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚13803465 NGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚60 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 5494538240 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚16245032 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚16245034 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚16245032 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚322 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 5494538240 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 PDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654602240 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 831258624 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 1121714853888 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚59395584 G@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚60 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚41948160 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚322 EDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59053056 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚31001413 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚13803403 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 125829120 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚209 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚262 # _timestamp‚1.78665982875289e+09 EDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚262 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 1060731247104 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚120825309485568 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚4293 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 1121760986112 _runtime‚1177.283257667 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 5368709120 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 1058516961784 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚262 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 5494538240 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 5368709120 G@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚62 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚3692281 SFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚66776064 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚105559818950656 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚4295 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚41948160 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚10111184 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 122306526 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚322 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚119764561133568 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚30997118 H@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚665 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚14752086 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 4654602240 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚119764578238464 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚104438104096768 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 125829120 IBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚60 >5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5264 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚14756379 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17104896 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 BDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521206 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 1058475613716 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 trainer/global_step‚4399 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚120825250432512 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚31001413 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚16245034 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 4552982528 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚3692221 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚41348068 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚4293 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚119764561133568 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 125829120 D¤Þ„ Ø /Purity/2FRt/event_purity‚0.3790395259857178 *jet/accuracy_2_of_2‚0.1994962990283966 /particle/accuracy_1_of_2‚0.8984028100967407 +particle/specificity‚0.6521203517913818 .Purity/*FRt/FRt_purity‚0.45435601472854614 particle/accuracy_1_of_1‚1 2Purity/*FRt/event_proportion‚0.669805645942688 0Purity/1FRt/event_purity‚0.44533106684684753 )jet/accuracy_1_of_2‚0.544708251953125 7validation_average_jet_accuracy‚0.30189526081085205 +jet/accuracy_1_of_1‚0.29739081859588623 (particle/accuracy‚0.6442754864692688 epoch‚10 % _timestamp‚1.7866598339540527e+09 _runtime‚1182.484159151 0Purity/*FRt/event_purity‚0.44145357608795166 3Purity/1FRt/event_proportion‚0.6297432780265808 _step‚362 trainer/global_step‚4399 *loss/val_total_loss‚2.5694101744583344 ,Purity/2FRt/FRt_purity‚0.524150013923645 +particle/sensitivity‚0.6300382018089294 'particle/f_score‚0.5569472312927246 Purity/0FRt/event_purity‚1 3Purity/0FRt/event_proportion‚0.3301943838596344 .Purity/1FRt/FRt_purity‚0.44533106684684753 2Purity/2FRt/event_proportion‚0.040062315762043 /particle/accuracy_2_of_2‚0.6723677515983582 *validation_accuracy‚0.1994962990283966œh`ã… Ý 'particle/f_score‚0.5569472312927246 Purity/0FRt/event_purity‚1 3Purity/0FRt/event_proportion‚0.3301943838596344 .Purity/1FRt/FRt_purity‚0.44533106684684753 2Purity/2FRt/event_proportion‚0.040062315762043 /particle/accuracy_2_of_2‚0.6723677515983582 *validation_accuracy‚0.1994962990283966 /Purity/2FRt/event_purity‚0.3790395259857178 *jet/accuracy_2_of_2‚0.1994962990283966 /particle/accuracy_1_of_2‚0.8984028100967407 +particle/specificity‚0.6521203517913818 .Purity/*FRt/FRt_purity‚0.45435601472854614 particle/accuracy_1_of_1‚1 2Purity/*FRt/event_proportion‚0.669805645942688 0Purity/1FRt/event_purity‚0.44533106684684753 )jet/accuracy_1_of_2‚0.544708251953125 7validation_average_jet_accuracy‚0.30189526081085205 +jet/accuracy_1_of_1‚0.29739081859588623 (particle/accuracy‚0.6442754864692688 epoch‚10 % _timestamp‚1.7866598339540527e+09 _runtime‚1182.484159151 0Purity/*FRt/event_purity‚0.44145357608795166 3Purity/1FRt/event_proportion‚0.6297432780265808 _step‚362 trainer/global_step‚4399 *loss/val_total_loss‚2.5694101744583344 ,Purity/2FRt/FRt_purity‚0.524150013923645 +particle/sensitivity‚0.6300382018089294êP{¶–† ‚(:‹ ü‡ùÓ¬Ò“á gpu.0.gpu‚62.0 gpu.0.memory‚12, gpu.0.memoryAllocated‚7.0313262939453125) gpu.0.memoryAllocatedBytes‚ 6039863296 gpu.0.temp‚36.0 gpu.0.powerWatts‚186.114' gpu.0.enforcedPowerLimitWatts‚500.0 gpu.0.powerPercent‚37.2228 gpu.0.smClock‚1410 gpu.0.memoryClock‚1593$ gpu.0.correctedMemoryErrors‚1955# gpu.0.uncorrectedMemoryErrors‚0°G‰j‡‡ ‚(:ü ü‡ùÓÝ€Üã$ disk./.usageGB‚460.0281181335449 disk.sda2.in‚0 disk.sda2.out‚0 disk.sda3.in‚ 10623.21875 disk.sda.out‚ 23.94140625* proc.memory.percent‚14.363861083984375 network.recv‚ 2.449158e+06* disk./.usagePercent‚30.872872997242816 disk.sda1.out‚0% disk.nvme0n1.out‚23434.3427734375 disk.drbd2165.in‚0 disk.sda3.out‚ 23.94140625! disk.nvme0c0n1.in‚ 359.6171875$ memory_percent‚66.06426239013672 disk.dm-1.out‚ 23.94140625' disk.nvme0c0n1.out‚23434.3427734375 disk.sda.in‚ 10623.21875& proc.memory.availableMB‚ 5560.03125 proc.memory.rssMB‚2353.375 cpu‚26.941213854097295 network.sent‚ 5.922367e+06 disk.sda1.in‚0 disk.nvme0n1.in‚ 359.6171875 disk.dm-1.in‚ 10621.921875 disk.drbd2165.out‚0 disk.dm-0.in‚ 359.6171875" disk.dm-0.out‚23434.3427734375 proc.cpu.threads‚17,K&2ˆ - _wandbruntime‚1195  _runtime‚1195Úð¿Áʼn ¿ _step‚363 % lr-AdamW/pg1‚0.001347518552668041 % lr-AdamW/pg2‚0.001347518552668041 trainer/global_step‚4449 % _timestamp‚1.7866598470263517e+09 _runtime‚1195.556317347n%ÜnÊŠ Ä % _timestamp‚1.7866598470263517e+09 _runtime‚1195.556317347 _step‚363 % lr-AdamW/pg1‚0.001347518552668041 % lr-AdamW/pg2‚0.001347518552668041 trainer/global_step‚4449ëX<‰l2‹ - _wandbruntime‚1195  _runtime‚1195‘ zÿ½'Œ ·O L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 4654602240 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚4287 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚322 PCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚59049984 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚322 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚121445292900352 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚122521405622784 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚59395584 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚120726765993984 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 4517085184 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚41945088 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚105866716315648 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚16479737 M@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚59049984 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 1076154667520 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 E8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚31415828 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 5368709120 trainer/global_step‚4449 IDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 1073914171838 F?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚60 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚16479735 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17104896 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚107004591740416 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 KDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚60 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 1076112722432 KBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚4876 G>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4876 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚121445310005248 PIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚79 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚52426752 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚120726783033344 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚322 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17104896 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚31420117 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚3740476 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚209 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚105866695671808 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 5368709120 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 5368709120 IBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚60 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚121445292900352 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚10259843 _step‚364 F>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 1073872823818 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 125829120 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 1076112722432 UHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚73070592 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚262 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 5494538240 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚41945088 JBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚665 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚14000398 IBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚81 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚3740555 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚107004518669824 % _timestamp‚1.7866598470272245e+09 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 122905600 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚262 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 4654602240 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4289 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚14000317 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 5494538240 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚4287 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚121445310005248 IDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚122521405622784 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚59395584 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚120726765993984 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 4517085184 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚41945088 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚105866716315648 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚16479737 M@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚59049984 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 1076154667520 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 E8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚31415828 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 5368709120 trainer/global_step‚4449 IDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 1073914171838 F?DeviceStatsMonitor.on_train_batch_stê_ñËHart/segment.small_pool.peak‚60 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚16479735 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17104896 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚107004591740416 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 KDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚60 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 1076112722432 KBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚4876 G>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4876 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚121445310005248 PIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚79 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚52426752 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚120726783033344 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚322 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17104896 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚31420117 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚3740476 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚209 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚105866695671808 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 5368709120 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 5368709120 IBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚60 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚121445292900352 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚10259843 _step‚364 F>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 1073872823818 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 125829120 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 1076112722432 UHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚73070592 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚262 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 5494538240 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚41945088 JBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚665 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚14000398 IBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚81 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚3740555 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚107004518669824 % _timestamp‚1.7866598470272245e+09 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 122905600 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚262 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 4654602240 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4289 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚14000317 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 5494538240 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚4287 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚121445310005248 I8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 4517085184 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚122548018082304 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 1076343791104 E6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚322 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚121827055183950 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚322 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚14003355 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚105889763688448 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521206 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 1076343791104 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚16483288 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚14939380 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 5494538240 _runtime‚1195.724831152 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚3741309 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚107027882043904 D=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚60 NADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚59053056 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚10262046 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚14943673 NGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚65 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚59395584 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 DDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654602240 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚31422668 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚121827113571378 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚41348068 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 1076385739264 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17104896 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚41948160 SFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚68873216 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚105889743044608 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚322 NADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚58387428 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚262 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚16483290 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 1076385739264 I@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚4876 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚120752968826880 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 1074103396430 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 5368709120 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 5368709120 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚41948160 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚120752951787520 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 125829120 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚16483290 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚429 trainer/global_step‚4449 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚14939380 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚14003422 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚262 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚121471632343040 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚223 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚107027813170688 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 845178880 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚48229376 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17104896 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚31426963 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚4295 K>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59053056 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚122548018082304 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚20643840 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚322 % _timestamp‚1.7866598471945195e+09 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 122905600 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 4552982528 G@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚67 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 1074144744498 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 125829120 G@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚60 _step‚365 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 122306526 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 5368709120 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚121471615238144 H@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚665 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 >5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5264 PDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654602240 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚31422668 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚121827113571378 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚41348068 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 1076385739264 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17104896 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚41948160 SFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚68873216 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚105889743044608 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚322 NADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚58387428 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚262 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚16483290 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 1076385739264 I@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚4876 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚120752968826880 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 1074103396430 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 5368709120 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 5368709120 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚41948160 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚120752951787520 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 125829120 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚16483290 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚429 trainer/global_step‚4449 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚14939380 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚14003422 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚262 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚121471632343040 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚223 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚107027813170688 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 845178880 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚48229376 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17104896 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚31426963 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚4295 K>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59053056 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚122548018082304 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚20643840 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚322 % _timestamp‚1.7866598471945195e+09 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 122905600 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 4552982528 G@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚67 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 1074144744498 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 125829120 G@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚60 _step‚365 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 122306526 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 5368709120 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚121471615238144 H@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚665 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 >5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5264 P8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 4517085184 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚122548018082304 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 1076343791104 E6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚322 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚121827055183950 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚322 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚14003355 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚105889763688448 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521206 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 1076343791104 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚16483288 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚14939380 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 5494538240 _runtime‚1195.724831152 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚3741309 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚107027882043904 D=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚60 NADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚59053056 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚10262046 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚14943673 NGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚65 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚59395584 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 DDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚123849109278208 BDeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4289 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚15101195 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚223 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 F>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚122761426894848 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚122761426894848 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚322 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 4552982528 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚15105482 KBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚4876 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚108169143106560 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 4517085184 _step‚368 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 4552982528 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚107019064311808 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 G>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4876 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚429 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚20643840 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 5494538240 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚322 JBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚665 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚10370091 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚322 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚15101195 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚123849168328192 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 122905600 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚14151753 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚108169214080000 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 122306526 F?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚60 % _timestamp‚1.7866598597363029e+09 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 125829120 IDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚123849109278208 BDeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4289 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚15101195 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚223 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 F>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚122761426894848 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚122761426894848 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚322 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 4552982528 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚15105482 KBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚4876 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚108169143106560 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 4517085184 _step‚368 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 4552982528 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚107019064311808 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 G>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4876 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚429 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚20643840 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 5494538240 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚322 JBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚665 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚10370091 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚322 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚15101195 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚123849168328192 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 122905600 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚14151753 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚108169214080000 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 122306526ðvyhv2œ - _wandbruntime‚1208  _runtime‚1208“´°AÇ& ÁM EDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 D=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚60 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚108192505759744 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚123147906671358 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚31765420 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 831258624 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 5494538240 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚123875721737728 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚15108775 B9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚5264 G@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚81 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚108192430595072 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚20643840 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 1085706555106 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 1150318910464 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚41348068 trainer/global_step‚4499 I@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚4876 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚429 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚31765420 >5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5264 NADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚59053056 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17104896 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚41948160 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚16660940 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚16660940 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 1087930556928 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚14154788 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521206 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 5368709120 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚322 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚262 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚15104482 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚31769715 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚15104482 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚10372298 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚3782492 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚322 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚4295 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚122787732127744 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚15108775 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚59395584 B8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚322 H@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚665 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚123875721737728 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚16660938 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 1087972505088 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚41948160 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654602240 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 125829120 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚322 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 1087972505088 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚123147965058786 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 122306526 DDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59053056 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚262 SFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚75164672 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 4552982528 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚16660938 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 4517085184 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 125829120 E5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5264 NADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚59053056 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17104896 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚41948160 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚16660940 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚16660940 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 1087930556928 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚14154788 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521206 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 5368709120 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚322 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚262 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚15104482 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚31769715 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚15104482 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚10372298 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚3782492 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚322 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚4295 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚122787732127744 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚15108775 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚59395584 B8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚322 H@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚665 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚123875721737728 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚16660938 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 1087972505088 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚41948160 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654602240 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 125829120 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚322 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 1087972505088 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚123147965058786 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 SEDeviceStatsMonito*Sð¼r.on_train_batch_end/requested_bytes.small_pool.peak‚ 122306526 DDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59053056 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚262 SFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚75164672 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 4552982528 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚16660938 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 4517085184 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 125829120 EDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 D=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚60 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚108192505759744 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚123147906671358ññ<2Ÿ - _wandbruntime‚1208  _runtime‚1208Í,-r  ì 0loss/FRt1/assignment_loss‚0.9890433990724934 /loss/FRt1/detection_loss‚0.5966235162442974 %loss/total_loss‚2.904842191870139 epoch‚11 trainer/global_step‚4499 _step‚370 0loss/FRt2/assignment_loss‚0.8334259239397228 0loss/FRt2/detection_loss‚0.48574935261362534 $ _timestamp‚1.786659859907816e+09 _runtime‚1208.437686789qÝ‹þw¡ ñ $ _timestamp‚1.786659859907816e+09 _runtime‚1208.437686789 0loss/FRt1/assignment_loss‚0.9890433990724934 /loss/FRt1/detection_loss‚0.5966235162442974 %loss/total_loss‚2.904842191870139 epoch‚11 trainer/global_step‚4499 _step‚370 0loss/FRt2/assignment_loss‚0.8334259239397228 0loss/FRt2/detection_loss‚0.48574935261362534òE]4”¢ ‚(:‰ šˆùÓË¥Šá gpu.0.gpu‚55.0 gpu.0.memory‚11, gpu.0.memoryAllocated‚7.0313262939453125) gpu.0.memoryAllocatedBytes‚ 6039863296 gpu.0.temp‚35.0 gpu.0.powerWatts‚182.74' gpu.0.enforcedPowerLimitWatts‚500.0 gpu.0.powerPercent‚36.548 gpu.0.smClock‚1410 gpu.0.memoryClock‚1593$ gpu.0.correctedMemoryErrors‚1955# gpu.0.uncorrectedMemoryErrors‚0Tº£ ‚(:‚ šˆùÓøÛã disk.sda2.out‚0 network.recv‚ 2.505124e+06* disk./.usagePercent‚30.975740664613582% disk./.usageGB‚461.56092071533203 disk.nvme0c0n1.in‚ 406.21875( disk.nvme0c0n1.out‚23508.81982421875 disk.sda.in‚ 11683.3828125& disk.nvme0n1.out‚23508.81982421875 disk.sda3.out‚ 24.44140625 disk.sda1.in‚0 disk.dm-1.out‚ 24.44140625 disk.dm-0.in‚ 406.21875# disk.dm-0.out‚23508.81982421875$ memory_percent‚66.06166362762451) proc.memory.availableMB‚ 5560.45703125* proc.memory.percent‚14.363861083984375 disk.sda.out‚ 24.44140625 disk.sda1.out‚0 disk.sda2.in‚0 network.sent‚ 6.065116e+06 disk.sda3.in‚ 11683.3828125 disk.nvme0n1.in‚ 406.21875 disk.drbd2165.out‚0 proc.memory.rssMB‚2353.375 disk.dm-1.in‚ 11683.3828125 disk.drbd2165.in‚0 cpu‚26.944219833616245 proc.cpu.threads‚17@¬ò2¤ - _wandbruntime‚1221  _runtime‚1221󢥘Ç¥ Á & lr-AdamW/pg1‚0.0013401765401774431 & lr-AdamW/pg2‚0.0013401765401774431 trainer/global_step‚4549 % _timestamp‚1.7866598732678442e+09 _runtime‚1221.799903349 _step‚371PvÚÀ̦ Æ _step‚371 & lr-AdamW/pg1‚0.0013401765401774431 & lr-AdamW/pg2‚0.0013401765401774431 trainer/global_step‚4549 % _timestamp‚1.7866598732678442e+09 _runtime‚1221.799903349óu75¸2§ - _wandbruntime‚1221  _runtime‚1221Tšç½'¨ ·O TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚41945088 % _timestamp‚1.7866598732685597e+09 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 4654602240 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚3822222 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 1099328199168 _runtime‚1221.800002346 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚20643840 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚322 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 122905600 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚32105621 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚322 E8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚32101332 IBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚66 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 BDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 4628521206 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 M@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚59049984 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 125829120 G>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4876 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 1099328199168 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 1162335737344 PCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚58387380 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚124442341792618 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 5368709120 F>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚108171432951808 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚123345362386944 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚125176871983616 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4289 KBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚4876 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 F?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚60 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 4552982528 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚262 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚4287 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 4552982528 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 122905600 IDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚125176812933632 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚41945088 C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚4289 KDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚60 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚322 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚41348020 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 1099286254080 UHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚68876288 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚109333768689152 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 831258624 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚123345345347584 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚125176871983616 JBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚665 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚10480341 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚125176812933632 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 1099286254080 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚14302565 PIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚64 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚3822158 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 125829120 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚262 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 5368709120 IBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚60 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚16835037 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚15266297 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚16835035 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚48232448 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 5494538240 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚10480343 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚59395584 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚124442400179998 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 125829120 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 4654602240 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚124077543784448 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚322 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 trainer/global_step‚4549 @7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚5264 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17104896 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚15266297 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 1162383969792YüÂ'© ¼O D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 4628521206 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 M@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚59049984 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 125829120 G>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4876 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 1099328199168 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 1162335737344 PCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚58387380 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚124442341792618 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 5368709120 F>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚108171432951808 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚123345362386944 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚125176871983616 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4289 KBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚4876 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 F?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚60 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 4552982528 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚262 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚4287 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 4552982528 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 122905600 IDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚125176812933632 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚41945088 C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚4289 KDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚60 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚322 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚41348020 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 1099286254080 UHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚68876288 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚109333768689152 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 831258624 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚123345345347584 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚125176871983616 JBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚665 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚10480341 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚125176812933632 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 1099286254080 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚14302565 PIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚64 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚3822158 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 125829120 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚262 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 5368709120 IBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚60 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚16835037 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚15266297 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚16835035 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚48232448 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 5494538240 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚10480343 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚59395584 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚124442400179998 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 125829120 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 4654602240 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚124077543784448 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚322 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 trainer/global_step‚4549 @7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚5264 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17104896 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚15266297 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 1162383969792 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚41945088 % _timestamp‚1.7866598732685597e+09 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 4654602240 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚3822222 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 1099328199168 _runtime‚1221.800002346 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚20643840 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚322 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 122905600 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚32105621 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚322 E8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚32101332 IBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚66 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 BDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚32112467 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 G@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚60 D=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚60 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚41948160 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 1099517322752 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚10482548 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚16838590 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚123371548180480 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚14305489 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 4654602240 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚124103849017344 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚32108172 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 845178880 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 5368709120 $ _timestamp‚1.786659873444639e+09 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17104896 B9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚5264 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 trainer/global_step‚4549 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚15269584 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚41948160 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚14305565 SFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚73067520 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654602240 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚4293 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚16838590 B5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5264 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚109357129769472 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521206 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚322 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 1099559270912 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 4552982528 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 5368709120 H@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚665 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚223 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 122905600 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚4293 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚322 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 125829120 K>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59053056 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚32112467 DDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 _runtime‚1221.975074964 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚10482546Õ„âË&¬ ÅM H@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚665 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚223 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 122905600 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚4293 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚322 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 125829120 K>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59053056 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚32112467 DDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 _runtime‚1221.975074964 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚10482546 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚59395584 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 1162576377344 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚322 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚262 _step‚373 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚4295 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17104896 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 122905600 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚3822943 NADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚58387428 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 5494538240 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚125203366340096 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚3823017 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚52423680 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚124468816546194 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 4552982528 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 125829120 EDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚32112467 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 G@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚60 D=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚60 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚41948160 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 1099517322752 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚10482548 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚16838590 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚123371548180480 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚14305489 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 4654602240 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚124103849017344 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚32108172 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 845178880 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 5368709120 $ _timestamp‚1.786659873444639e+09 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17104896 B9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚5264 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 trainer/global_step‚4549 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚15269584 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚41948160 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚14305565 SFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚73067520 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654602240 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚4293 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚16838590 B5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5264 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚109357129769472 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521206 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚322 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 1099559270912 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 4552982528 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 5368709120õ Ü2­ - _wandbruntime‚1221  _runtime‚1221üÔÓt® î 0loss/FRt1/assignment_loss‚0.8796335457415986 0loss/FRt2/assignment_loss‚0.8274470748240769 trainer/global_step‚4549 _runtime‚1221.975791909 0loss/FRt1/detection_loss‚0.48774823797381894 /loss/FRt2/detection_loss‚0.5298258194629583 &loss/total_loss‚2.7246546780024525 epoch‚11 % _timestamp‚1.7866598734459577e+09 _step‚374|à×y¯ ó 0loss/FRt1/assignment_loss‚0.8796335457415986 0loss/FRt2/assignment_loss‚0.8274470748240769 trainer/global_step‚4549 _runtime‚1221.975791909 0loss/FRt1/detection_loss‚0.48774823797381894 /loss/FRt2/detection_loss‚0.5298258194629583 &loss/total_loss‚2.7246546780024525 epoch‚11 % _timestamp‚1.7866598734459577e+09 _step‚374ö‹Ûx–° ‚(:‹ ©ˆùÓÁãˆá gpu.0.gpu‚64.0 gpu.0.memory‚12, gpu.0.memoryAllocated‚7.0313262939453125) gpu.0.memoryAllocatedBytes‚ 6039863296 gpu.0.temp‚35.0 gpu.0.powerWatts‚229.511' gpu.0.enforcedPowerLimitWatts‚500.0 gpu.0.powerPercent‚45.9022 gpu.0.smClock‚1410 gpu.0.memoryClock‚1593$ gpu.0.correctedMemoryErrors‚1955# gpu.0.uncorrectedMemoryErrors‚0*rˆ± ‚(:‚ ©ˆùÓÝ£éâ disk.drbd2165.out‚0" disk.dm-0.out‚29556.3505859375" disk.nvme0c0n1.in‚ 439.83984375 disk.sda1.out‚0 proc.memory.rssMB‚2353.375 proc.cpu.threads‚17 network.sent‚ 6.138e+06* disk./.usagePercent‚31.302939029982944 disk.drbd2165.in‚0 disk.dm-0.in‚ 439.83984375 disk.sda3.in‚ 11683.3828125' disk.nvme0c0n1.out‚29556.3505859375 disk.sda.in‚ 11683.3828125 disk.sda.out‚ 24.87109375 disk.nvme0n1.in‚ 439.83984375% disk.nvme0n1.out‚29556.3505859375 disk.sda3.out‚ 24.87109375 disk.sda2.in‚0 disk.sda2.out‚0$ memory_percent‚66.06507301330566( proc.memory.availableMB‚ 5559.8984375* proc.memory.percent‚14.363861083984375 network.recv‚ 2.535564e+06 disk.dm-1.in‚ 11683.3828125 disk.sda1.in‚0 cpu‚26.94551118336791$ disk./.usageGB‚466.4364128112793 disk.dm-1.out‚ 24.87109375Ûä"2² - _wandbruntime‚1234  _runtime‚1234QïK³Ç³ Á _step‚375 & lr-AdamW/pg1‚0.0013364485565242347 & lr-AdamW/pg2‚0.0013364485565242347 trainer/global_step‚4599 % _timestamp‚1.7866598860856829e+09 _runtime‚1234.615683672®¤ÌÌ´ Æ & lr-AdamW/pg1‚0.0013364485565242347 & lr-AdamW/pg2‚0.0013364485565242347 trainer/global_step‚4599 % _timestamp‚1.7866598860856829e+09 _runtime‚1234.615683672 _step‚375÷ %j82µ - _wandbruntime‚1234  _runtime‚1234^¬Da¼'¶ ¶O WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 1110873019904 BDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4876 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚41348020 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 125829120 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚322 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 1110873019904 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4289 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚3862708 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚17012687 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚322 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚262 $ _timestamp‚1.786659886086637e+09 C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚4289 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚17012685 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚3862640 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 F?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚60 @7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚5264 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 4552982528 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚32448373 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚109323822235648 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚125763193280026 trainer/global_step‚4599 _runtime‚1234.619874225 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚15435686 D;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚5264 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 1174643600896 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚322 KBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚4876 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚59395584 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 5368709120 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚262 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚124654652063744 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚41945088 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 KDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚60 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 831258624 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚125763251667406 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17104896 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚14453231 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚126504516589056 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 5368709120 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 4654602240 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 1110914964992 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚223 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 5494538240 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚14453301 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚209 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 4628521206 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 5368709120 IDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚126504575639040 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 122306526 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 122905600 IBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚70 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 M@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚59049984ÔƒqÖ· »O PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚15435686 D;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚5264 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 1174643600896 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚322 KBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚4876 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚59395584 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 5368709120 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚262 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚124654652063744 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚41945088 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 KDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚60 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 831258624 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚125763251667406 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17104896 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚14453231 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚126504516589056 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 5368709120 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 4654602240 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 1110914964992 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚223 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 5494538240 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚14453301 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚209 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 4628521206 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 5368709120 IDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚126504575639040 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 122306526 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 122905600 IBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚70 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 M@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚59049984 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 1110873019904 B¾ë_bytes.all.peak‚ 845178880 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚262 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 1174595368448 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚125393660674048 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 1110914964992 JBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚665 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚4287 PCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚58387380 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 5494538240 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚17012685 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚4287 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17104896 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚41945088 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚15431399 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚10590593 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚110498465836544 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 4517085184 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚10590591 PCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚59049984 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚125393643569152 IDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4876 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚41348020 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 125829120 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚322 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 1110873019904 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4289 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚3862708 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚17012687 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚322 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚262 $ _timestamp‚1.786659886086637e+09 C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚4289 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚17012685 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚3862640 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 F?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚60 @7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚5264 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 4552982528 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚32448373 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚109323822235648 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚125763193280026 trainer/global_step‚4599 _runtime‚1234.619874225ø¢Ñ&F2¸ - _wandbruntime‚1234  _runtime‚1234.ºõÇ&¹ ÁM T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚126531129048576 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 1111146036736 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚17016238 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 1108788828254 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 122306526 trainer/global_step‚4599 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚41948160 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚46132224 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 4517085184 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚15438979 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 831258624 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚59395584 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 D=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚60 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17104896 NGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚69 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚322 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚126531129048576 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚125419983011840 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚3863413 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 G@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚60 H@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚665 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚14456209 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 5494538240 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚4293 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 1108830176322 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 125829120 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚125419983011840 E8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 5368709120 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚322 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚125419965906944 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654602240 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚209 NADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚58387428 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚15434686 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 5368709120 % _timestamp‚1.7866598862479665e+09 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚32455219 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 4552982528 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17104896 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚10592798 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 1111146036736 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚124680820817920 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚41348068 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚262 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 122905600 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚125419965906944 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 4552982528 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚32450924 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚429 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚41948160 B9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚5264 I@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚4876 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚110521754403328 PDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚32455219 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚10592796 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚110521687627264 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 4654602240 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 1111104088576 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚15434686 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚109346848964608 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚17016240 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚3863482 B5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5264 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 122905600 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 5368709120 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 125829120 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚32450924 SFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚66776064 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521206 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚262 DDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 _step‚377 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 K>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59053056 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚322 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚125789609646174 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚124680837857280 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚4295 EDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 _runtime‚1234.778290261 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚126531069995520 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚14456280 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚20643840 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 5494538240 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 5494538240 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚4293 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 1111104088576 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 1174838662656 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚322’ŽcÌ&º ÆM QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 4552982528 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17104896 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚10592798 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 1111146036736 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚124680820817920 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚41348068 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚262 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 122905600 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚125419965906944 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 4552982528 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚32450924 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚429 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚41948160 B9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚5264 I@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚4876 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚110521754403328 PDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚32455219 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚10592796 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚110521687627264 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 4654602240 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 1111104088576 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚15434686 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚109346848964608 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚17016240 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚3863482 B5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5264 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 122905600 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 5368709120 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 125829120 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚32450924 SFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚66776064 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521206 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚262 DDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 _step‚377 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 K>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59053056 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚322 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚125789609646174 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚124680837857280 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚4295 EDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 _runtime‚1234.778290261 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚126531069995520 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚14456280 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚20643840 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 5494538240 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 5494538240 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚4293 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 1111104088576 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 1174838662656 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚322 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚126531129048576 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 1111146036736 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚17016238 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 1108788828254 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 122306526 trainer/global_step‚4599 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚41948160 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚46132224 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 4517085184 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚15438979 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 831258624 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚59395584 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 D=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚60 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17104896 NGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚69 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚322 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚126531129048576 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚125419983011840 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚3863413 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 G@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚60 H@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚665 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚14456209 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 5494538240 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚4293 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 1108830176322 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 125829120 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚125419983011840 E8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 5368709120 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚322 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚125419965906944 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654602240 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚209 NADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚58387428 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚15434686 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 5368709120 % _timestamp‚1.7866598862479665e+09 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚32455219ùÖ¦¿ 2» - _wandbruntime‚1234  _runtime‚1234NÂOÂr¼ ì %loss/total_loss‚2.292263999001666 epoch‚11 % _timestamp‚1.7866598862489374e+09 _runtime‚1234.778926222 0loss/FRt1/assignment_loss‚0.6571303068121974 0loss/FRt2/assignment_loss‚0.6191069808156637 /loss/FRt1/detection_loss‚0.5266640706433856 trainer/global_step‚4599 _step‚378 /loss/FRt2/detection_loss‚0.4893626407304192ú×w½ ñ %loss/total_loss‚2.292263999001666 epoch‚11 % _timestamp‚1.7866598862489374e+09 _runtime‚1234.778926222 0loss/FRt1/assignment_loss‚0.6571303068121974 0loss/FRt2/assignment_loss‚0.6191069808156637 /loss/FRt1/detection_loss‚0.5266640706433856 trainer/global_step‚4599 _step‚378 /loss/FRt2/detection_loss‚0.4893626407304192úÐNQˆ•¾ ‚(:Š ¸ˆùÓ×Øþà gpu.0.gpu‚27.0 gpu.0.memory‚3, gpu.0.memoryAllocated‚7.0313262939453125) gpu.0.memoryAllocatedBytes‚ 6039863296 gpu.0.temp‚36.0 gpu.0.powerWatts‚220.122' gpu.0.enforcedPowerLimitWatts‚500.0 gpu.0.powerPercent‚44.0244 gpu.0.smClock‚1410 gpu.0.memoryClock‚1593$ gpu.0.correctedMemoryErrors‚1955# gpu.0.uncorrectedMemoryErrors‚0'–Ðþ¿ ‚(:… ¸ˆùÓÑÀÛã disk.sda3.in‚ 11683.3828125 disk.sda2.in‚0 disk.nvme0n1.in‚ 441.6796875& disk.nvme0n1.out‚30173.27099609375 disk.dm-1.in‚ 11683.3828125 disk.drbd2165.in‚0 disk.drbd2165.out‚0 disk.sda2.out‚0 disk.sda3.out‚ 25.0859375! disk.nvme0c0n1.in‚ 441.6796875 disk.sda.out‚ 25.0859375# disk.dm-0.out‚30173.27099609375 proc.memory.rssMB‚2353.375* proc.memory.percent‚14.363861083984375 cpu‚26.947669970561734( disk.nvme0c0n1.out‚30173.27099609375 disk.sda1.out‚0 disk.dm-0.in‚ 441.6796875$ memory_percent‚66.06471538543701 proc.cpu.threads‚17 network.recv‚ 2.565248e+06* disk./.usagePercent‚31.303314593035676% disk./.usageGB‚466.44200897216797 disk.sda.in‚ 11683.3828125 disk.sda1.in‚0 disk.dm-1.out‚ 25.0859375) proc.memory.availableMB‚ 5559.95703125 network.sent‚ 6.210169e+06r¢z˜2À - _wandbruntime‚1246  _runtime‚1246Ñ‹§VÅÁ ¿ $ _timestamp‚1.786659898284405e+09 _runtime‚ 1246.81444059 _step‚379 & lr-AdamW/pg1‚0.0013326829063761091 & lr-AdamW/pg2‚0.0013326829063761091 trainer/global_step‚4649ö+vóÊ Ä & lr-AdamW/pg1‚0.0013326829063761091 & lr-AdamW/pg2‚0.0013326829063761091 trainer/global_step‚4649 $ _timestamp‚1.786659898284405e+09 _runtime‚ 1246.81444059 _step‚379ûÕãÒ2à - _wandbruntime‚1246  _runtime‚1246˜B»Ä ·O PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚17190335 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚20643840 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 4628521206 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 1120120066250 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 1122459785728 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚41945088 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4289 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚125963941740544 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 trainer/global_step‚4649 F?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚60 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚4287 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17104896 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚10700841 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 4517085184 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚15600788 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚59395584 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚41348020 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 5368709120 IDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 5494538240 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚322 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚262 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚17190337 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 1186849879552 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 1122459785728 D;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚5264 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 G>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4876 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚127832279294464 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚111663088987648 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 PCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚58387380 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚127084044767434 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚126709777563648 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚15596501 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚14604124 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 845178880 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 125829120 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 _runtime‚1246.815622431 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚111663020111360 60DeviceStatsM| “Eonitor.on_train_batch_start/num_ooms‚0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚262 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚127832220244480 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 4552982528 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 1120161414270 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 5368709120 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚322 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 E8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚32786836 F>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 1122501730816 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 831258624 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚322 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 122905600 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚125963924701184 _step‚380 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚17190335 M@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚59049984 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚10700843 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚223 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚110476170231808 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚3903281 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 1186898112000 KDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚60 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚429 UHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚68876288 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 5368709120 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 125829120 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚127832220244480 KBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚4876 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚3903206 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚110476190875648 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚322 C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚4289 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 4552982528 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚4287 IBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚77 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚209 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 5494538240 @7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚5264 JBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚665 BDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4876 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚127832279294464 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚111663088987648 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 PCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚58387380 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚127084044767434 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚126709777563648 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚15596501 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚14604124 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 845178880 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 125829120 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 _runtime‚1246.815622431 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚111663020111360 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚262 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚127832220244480 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 4552982528 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 1120161414270 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 5368709120 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚322 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 E8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚32786836 F>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 1122501730816 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 831258624 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚322 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 122905600 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚125963924701184 _step‚380 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚17190335 M@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚59049984 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚10700843 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚223 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚110476170231808 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚3903281 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 1186898112000 KDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚60 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚429 UHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚68876288 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 5368709120 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 125829120 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚127832220244480 KBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚4876 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚3903206 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚110476190875648 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚322 C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚4289 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 4552982528 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚4287 IBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚77 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚209 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 5494538240 @7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚5264 JBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚665 BDeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4289 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚125963941740544 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 trainer/global_step‚4649 F?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚60 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚4287 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17104896 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚10700841 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 4517085184 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚15600788 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚59395584 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚41348020 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 5368709120 IDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 5494538240 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚322 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚262 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚17190337 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 1186849879552 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 1122459785728üšLH 2Æ - _wandbruntime‚1246  _runtime‚1246N(ÂåÆ&Ç ÀM UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17104896 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 122905600 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚3904089 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚17193890 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17104896 I@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚4876 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚223 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚209 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚111686316512768 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚20643840 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 1120391986930 IBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚60 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚32797971 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚4295 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 1122690854400 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚44035072 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚262 B9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚5264 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚127110461133582 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 4517085184 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 5494538240 $ _timestamp‚1.786659898447503e+09 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654602240 G@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚64 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚3904027 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚41348068 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚14607137 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 845178880 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚322 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚111686381191680 SFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚64678912 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚59395584 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 5494538240 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚4293 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 4654602240 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚126736099901440 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚10703048 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 831258624 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚127858832704000 K>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59053056 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 DDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚41948160 EDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521206 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 5368709120 trainer/global_step‚4649 _step‚381 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 4552982528 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 125829120 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚32793676 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚17193888 E6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚322 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚41948160 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 1187142943232 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 _runtime‚1246.977870404 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚127858773650944 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚15599788 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 B5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5264 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 5494538240 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚15599788 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚14607073 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚110499238248448 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 1187098908160 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚125990110494720 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 125829120 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚322 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚262 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚127858832704000 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚126736082796544 G@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚60 D=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚60 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚4293 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚429 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚10703046 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚127110519521010 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚262 NADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚59053056 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚32797971ÏGÔ!È ÅM [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚126736099901440 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚126736082796544 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 1120350638862 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚41948160 EDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521206 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 5368709120 trainer/global_step‚4649 _step‚381 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 4552982528 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 125829120 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚32793676 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚17193888 E6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚322 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚41948160 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 1187142943232 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 _runtime‚1246.977870404 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚127858773650944 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚15599788 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 B5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5264 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 5494538240 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚15599788 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚14607073 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚110499238248448 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 1187098908160 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚125990110494720 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 125829120 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚322 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚262 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚127858832704000 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚126736082796544 G@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚60 D=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚60 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚4293 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚429 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚10703046 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚127110519521010 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚262 NADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚59053056 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚32797971 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17104896 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 122905600 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚3904089 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚17193890 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17104896 I@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚4876 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚223 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚209 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚111686316512768 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚20643840 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 1120391986930 IBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚60 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚32797971 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚4295 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 1122690854400 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚44035072 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚262 B9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚5264 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚127110461133582 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 4517085184 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 5494538240 $ _timestamp‚1.786659898447503e+09 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654602240 G@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚64 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚3904027 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚41348068 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚14607137 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 845178880 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚322 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚111686381191680 SFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚64678912 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚59395584 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 5494538240 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚4293 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 4654602240 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚126736099901440 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚10703048 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 831258624 KEDeviceStatsMonitor/–‹ª.on_train_batch_end/reserved_bytes.large_pool.freed‚0 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚127858832704000 K>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59053056 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 DDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚41945088 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚223 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 5494538240 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 5494538240 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚15761603 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 122905600 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 4654602240 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 1199133320704 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 5368709120 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 4552982528 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚128025877348352 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚17367985 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚127273231417344 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 5368709120 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 125829120 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚129159923899904 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚33133877 PIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚76 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 IDeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4289 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 845178880 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚111628538871808 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚46135296 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚59395584 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 G>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4876 BDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚262 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚128025894453248 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17104896 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚10811093 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚3944182 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 1131723224878 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚322 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 1134046551552 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 KBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚4876 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚14755197 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚111628559515648 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 831258624 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚129159982949888 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚129159923899904 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚112827738971648 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 5368709120 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚322 KDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚60 UHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚66779136 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 _runtime‚1259.178896114 D;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚5264 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚128404954642222 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 1131681876858 % _timestamp‚1.7866599106450841e+09 @7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚5264 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚17367987 JBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚665 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 4628521206 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 125829120—Ù¯¹Â'Ñ ¼O [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 1134088496640 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 4517085184 IBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚78 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚17367987 M@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚59049984 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17104896 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 1199179456000 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 5494538240 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚262 trainer/global_step‚4699 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚128025894453248 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 125829120 C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚4289 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 4552982528 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚41945088 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚127273214377984 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚262 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 F?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚60 E8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚33129588 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚322 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 1134088496640 PCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚58387380 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4289 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 845178880 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚111628538871808 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚46135296 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚59395584 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 G>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4876 BDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚262 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚128025894453248 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17104896 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚10811093 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚3944182 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 1131723224878 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚322 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 1134046551552 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 KBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚4876 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚14755197 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚111628559515648 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 831258624 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚129159982949888 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚129159923899904 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚112827738971648 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 5368709120 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚322 KDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚60 UHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚66779136 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 _runtime‚1259.178896114 D;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚5264 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚128404954642222 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 1131681876858 % _timestamp‚1.7866599106450841e+09 @7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚5264 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚17367987 JBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚665 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 4628521206 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 125829120 PCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚59049984 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚112827672192512 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚20643840 _step‚384 F>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚41945088 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚223 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 5494538240 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 5494538240 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚15761603 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 122905600 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 4654602240 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 1199133320704 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 5368709120 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 4552982528 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚128025877348352 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚17367985 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚127273231417344 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 5368709120 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 125829120 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚129159923899904 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚33133877 PIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚76 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 I5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5264 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 122905600 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654602240 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚128052216791040 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚223 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚41948160 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚262 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17104896 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 1134319568384 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚15769183 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 845178880 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 122306526 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 5368709120 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚17371540 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚322 D=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚60 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚15764890 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚17371540 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 4517085184 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 B9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚5264 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚50326528 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚128431371008418 _step‚385 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 1134277620224 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚209 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚15769183 NGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚80 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚3944953 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 1199424811520 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚262 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚4295 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 4552982528 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚112850960729600 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 125829120 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚4293 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚14758331 NADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚58387428 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚41348068 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 5368709120 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚112851031699968 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚20643840 BDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59053056 EDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚14758249 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚127299400171520 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚322 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚262 D8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 5494538240 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚129186536359424 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 4654602240 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚129186477306368 H@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚665 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 P6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚322 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 1131912449470 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚41948160 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521206 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚127299417210880 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚3945033 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚128431312620990 # _timestamp‚1.78665991080848e+09 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚15764890 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚322 G@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚60 G@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚82 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 I@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚4876 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 5494538240 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 125829120 _runtime‚1259.338804596 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 4552982528 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚33136428 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 831258624 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 125829120 trainer/global_step‚4699 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚33140723 EDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚33140723 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0ºsålÊ&Ô ÄM NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚15769183 NGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚80 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚3944953 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 1199424811520 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚262 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚4295 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 4552982528 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚112850960729600 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 125829120 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚4293 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚14758331 NADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚58387428 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚41348068 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 5368709120 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚112851031699968 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚20643840 BDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59053056 EDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚14758249 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚127299400171520 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚322 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚262 D8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 5494538240 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚129186536359424 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 4654602240 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚129186477306368 H@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚665 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 P6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚322 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 1131912449470 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚41948160 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521206 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚127299417210880 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚3945033 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚128431312620990 # _timestamp‚1.78665991080848e+09 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚15764890 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚322 G@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚60 G@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚82 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 I@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚4876 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 5494538240 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 125829120 _runtime‚1259.338804596 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 4552982528 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚33136428 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 831258624 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 125829120 trainer/global_step‚4699 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚33140723 EDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚33140723 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 >5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5264 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 122905600 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654602240 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚128052216791040 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚223 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚41948160 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚262 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17104896 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 1134319568384 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚15769183 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 845178880 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 122306526 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 5368709120 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚17371540 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚322 D=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚60 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚15764890 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚17371540 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 4517085184 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 B9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚5264 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚50326528 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚128431371008418 _step‚385 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 1134277620224 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚209 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0ü8µ2Õ - _wandbruntime‚1259  _runtime‚1259 9e¥sÖ í 0loss/FRt2/detection_loss‚0.49139543522214574 epoch‚11 trainer/global_step‚4699 % _timestamp‚1.7866599108094292e+09 _runtime‚1259.339232806 /loss/FRt1/detection_loss‚0.5893805765061488 %loss/total_loss‚2.671921960422347 _step‚386 0loss/FRt1/assignment_loss‚0.9278381801349126 0loss/FRt2/assignment_loss‚0.6633077685591402Ê™ôx× ò trainer/global_step‚4699 % _timestamp‚1.7866599108094292e+09 _runtime‚1259.339232806 /loss/FRt1/detection_loss‚0.5893805765061488 %loss/total_loss‚2.671921960422347 _step‚386 0loss/FRt1/assignment_loss‚0.9278381801349126 0loss/FRt2/assignment_loss‚0.6633077685591402 0loss/FRt2/detection_loss‚0.49139543522214574 epoch‚11‚[Ié. Ø ‚(:• LjùÓ³Çá gpu.0.gpu‚58.0 gpu.0.memory‚10, gpu.0.memoryAllocated‚7.0313262939453125) gpu.0.memoryAllocatedBytes‚ 6039863296 gpu.0.temp‚36.0 gpu.0.powerWatts‚141.74' gpu.0.enforcedPowerLimitWatts‚500.0) gpu.0.powerPercent‚28.348000000000003 gpu.0.smClock‚1410 gpu.0.memoryClock‚1593$ gpu.0.correctedMemoryErrors‚1955# gpu.0.uncorrectedMemoryErrors‚0B~XÙ ‚(:‚ LjùÓý¶©ã disk.drbd2165.in‚0' disk.nvme0c0n1.out‚30182.6181640625 disk.sda.out‚ 25.59765625) proc.memory.availableMB‚ 5559.99609375 disk.nvme0n1.in‚ 441.6796875% disk.nvme0n1.out‚30182.6181640625 disk.dm-1.in‚ 11683.3828125 disk.drbd2165.out‚0" disk.dm-0.out‚30182.6181640625 disk.sda3.in‚ 11683.3828125 disk.sda3.out‚ 25.59765625! disk.nvme0c0n1.in‚ 441.6796875 network.recv‚ 2.59203e+06 disk.sda1.in‚0 disk.dm-0.in‚ 441.6796875 disk.sda.in‚ 11683.3828125$ memory_percent‚66.06447696685791 proc.memory.rssMB‚2353.375 proc.cpu.threads‚17 network.sent‚ 6.27961e+06 disk.sda2.in‚0 disk.dm-1.out‚ 25.59765625* proc.memory.percent‚14.363861083984375 cpu‚26.949641009983978* disk./.usagePercent‚31.303337633713756$ disk./.usageGB‚466.4423522949219 disk.sda1.out‚0 disk.sda2.out‚0f©ðK2Ú - _wandbruntime‚1271  _runtime‚1271§²ÇÛ Á & lr-AdamW/pg2‚0.0013250395764415107 trainer/global_step‚4749 % _timestamp‚1.7866599234238436e+09 _runtime‚1271.953913742 _step‚387 & lr-AdamW/pg1‚0.0013250395764415107²†SÌÜ Æ % _timestamp‚1.7866599234238436e+09 _runtime‚1271.953913742 _step‚387 & lr-AdamW/pg1‚0.0013250395764415107 & lr-AdamW/pg2‚0.0013250395764415107 trainer/global_step‚4749ƒhšQ2Ý - _wandbruntime‚1271  _runtime‚1271¾»5½'Þ ·O NFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚223 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 % _timestamp‚1.7866599234248588e+09 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 1145675262464 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚128582504054784 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 125829120 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 125829120 IDeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4289 D;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚5264 KDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚60 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚17545635 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚322 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚129725806129630 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 PIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚75 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 1143285035486 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚262 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚4287 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚3984992 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚209 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 831258624 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚17545637 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 trainer/global_step‚4749 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚15926705 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚4289 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚15930992 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚113992316142592 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 4628521206 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 122905600 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚15930992 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚3984917 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 845178880 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 5494538240 IBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚60 G>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4876 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚129342011342848 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚14906258 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚129342011342848 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 4552982528 IDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 M@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚59049984 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚130487627555328 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚20643840 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚41348020 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 122306526 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 KBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚4876 BDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 M@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚59049984 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚130487627555328 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚20643840 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚41348020 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 122306526 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 KBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚4876 BDeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4289 D;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚5264 KDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚60 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚17545635 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚322 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚129725806129630 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 PIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚75 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 1143285035486 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚262 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚4287 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚3984992 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚209 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 831258624 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚17545637 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 trainer/global_step‚4749 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚15926705 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚4289 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚15930992 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚113992316142592 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 4628521206 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 122905600 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚15930992 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚3984917 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 845178880 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 5494538240 IBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚60 G>DeviceStatsMonitor.on_train_batch_start/active.small_pofÉGškol.peak‚4876 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚129342011342848 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚14906258 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚129342011342848 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 4552982528 I8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚128608689848320 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654602240 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚33483475 H@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚665 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 5494538240 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 trainer/global_step‚4749 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚17549190 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚114015676084224 NADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚59053056 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚129368316575744 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚14909300 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 NADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚58387428 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 5368709120 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 1145864386048 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 831258624 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚20643840 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 125829120 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚322 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚3985830 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚128608706887680 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚48229376 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚129752222495826 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚41948160 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 122905600 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚3985754 _runtime‚1272.125003571 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 1145906334208 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 122905600 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚429 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 845178880 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 1211652326400 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚322 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚15929992 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521206 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚262 % _timestamp‚1.7866599235946949e+09 G@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚78 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17104896 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚33483475 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚209 >5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5264 D6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚322 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚129752164108398 K>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59053056 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 1145864386048 EDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚15934285 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚112803975528448 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 125829120 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚130514180961792 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚129368333680640 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 4552982528 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚129368316575744 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚322 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚61478912 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 B8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚128608689848320 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654602240 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚33483475 H@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚665 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 5494538240 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 trainer/global_step‚4749 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚17549190 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚114015676084224 NADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚59053056 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚129368316575744 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚14909300 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 NADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚58387428 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 5368709120 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 1145864386048 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 831258624 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚20643840 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 125829120 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚322 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚3985830 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚128608706887680 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚48229376 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚129752222495826 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚41948160 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 122905600 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚3985754 _runtime‚1272.125003571 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 1145906334208 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 122905600 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚429 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 845178880 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 1211652326400 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚322 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚15929992 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521206 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚262 % _timestamp‚1.7866599235946949e+09 G@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚78 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17104896 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚33483475 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚209 >5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5264 D6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚322 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚129752164108398 K>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59053056 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 1145864386048 EDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚15934285 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚112803975528448 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 125829120 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚130514180961792 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚129368333680640 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 4552982528 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚129368316575744 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚322…Ý•^!2ã - _wandbruntime‚1272  _runtime‚12725ácCtä î &loss/total_loss‚2.7149389920750107 epoch‚11 trainer/global_step‚4749 % _timestamp‚1.7866599235958135e+09 0loss/FRt2/assignment_loss‚0.6660107178873486 _runtime‚1272.125619214 _step‚390 0loss/FRt1/assignment_loss‚0.9623088984673642 /loss/FRt1/detection_loss‚0.5940250994150261 0loss/FRt2/detection_loss‚0.49259427630527175ÉRwwyå ó 0loss/FRt2/assignment_loss‚0.6660107178873486 _runtime‚1272.125619214 _step‚390 0loss/FRt1/assignment_loss‚0.9623088984673642 /loss/FRt1/detection_loss‚0.5940250994150261 0loss/FRt2/detection_loss‚0.49259427630527175 &loss/total_loss‚2.7149389920750107 epoch‚11 trainer/global_step‚4749 % _timestamp‚1.7866599235958135e+09†Ç$o@æ ‚(:‚ ÖˆùÓá¨ã disk.sda2.in‚0# memory_percent‚66.0646915435791( proc.memory.availableMB‚ 5559.9609375 proc.cpu.threads‚17! disk.nvme0c0n1.in‚ 441.6796875 disk.sda.out‚ 25.79296875 disk.sda1.out‚0 disk.sda2.out‚0% disk.nvme0n1.out‚30192.8427734375 disk.dm-1.out‚ 25.79296875 disk.drbd2165.in‚0 disk.drbd2165.out‚0 disk.sda3.in‚ 11683.3828125' disk.nvme0c0n1.out‚30192.8427734375 disk.nvme0n1.in‚ 441.6796875 disk.dm-0.in‚ 441.6796875" disk.dm-0.out‚30192.8427734375 proc.memory.rssMB‚2353.375* proc.memory.percent‚14.363861083984375 network.sent‚ 6.369097e+06* disk./.usagePercent‚31.303274399852803% disk./.usageGB‚466.44141006469727 disk.sda1.in‚0 disk.dm-1.in‚ 11683.3828125 cpu‚26.95097909686778 network.recv‚ 2.617663e+06 disk.sda3.out‚ 25.79296875 disk.sda.in‚ 11683.3828125(§dV•ç ‚(:Š ÖˆùÓÎö›ú gpu.0.gpu‚64.0 gpu.0.memory‚12, gpu.0.memoryAllocated‚7.0313262939453125) gpu.0.memoryAllocatedBytes‚ 6039863296 gpu.0.temp‚36.0 gpu.0.powerWatts‚222.995' gpu.0.enforcedPowerLimitWatts‚500.0 gpu.0.powerPercent‚44.599 gpu.0.smClock‚1410 gpu.0.memoryClock‚1593$ gpu.0.correctedMemoryErrors‚1955# gpu.0.uncorrectedMemoryErrors‚0¶$íl2è - _wandbruntime‚1284  _runtime‚1284Íw†YÇé Á & lr-AdamW/pg2‚0.0013211623875718367 trainer/global_step‚4799 % _timestamp‚1.7866599359940224e+09 _runtime‚1284.524021611 _step‚391 & lr-AdamW/pg1‚0.0013211623875718367u„ŒzÌê Æ & lr-AdamW/pg1‚0.0013211623875718367 & lr-AdamW/pg2‚0.0013211623875718367 trainer/global_step‚4799 % _timestamp‚1.7866599359940224e+09 _runtime‚1284.524021611 _step‚391‡ÂSt&2ë - _wandbruntime‚1284  _runtime‚1284Ea(|6"ì ¶O UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚131815331210752 IDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚131815331210752 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚130658111127552 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚429 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚131046599229658 JBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚665 C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚4289 M@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚59049984 PCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚59049984 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚33819381 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 831258624 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚130658111127552 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 1157262028288 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4289 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚115157027248128 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 125829120 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚131815390260736 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚17723287 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚115156962566144 BDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 4654602240 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚130658128232448 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚4287 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚4026110 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 4517085184 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 5494538240 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚223 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚209 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 125829120 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚262 IBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚60 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚130658128232448 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17104896 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚44038144 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚262 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 4552982528 D;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚5264 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚113933296795648 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 4654602240 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚16091807 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚113933276151808 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚17723285 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17104896 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚322 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 1154846846094 F?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚60 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚4026175 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 5494538240 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 1157262028288 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 4552982528 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 5494538240 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚11031591 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚20643840 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 KBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚4876 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 1223686414336 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚61478912 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 1154805498074 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 122905600 _runtime‚ 1284.52888259 UHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚64681984 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 IBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚67 60DeviceStatsM^¹n.†onitor.on_train_batch_start/num_ooms‚0 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚129891810770944 KDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚60 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 G>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4876 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 _step‚392 E8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚33815092 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 845178880 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚41348020 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 122905600 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚16096094 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚131046657617038 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 1157220083200 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚11031593 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0ezÐÉÁ'í »O RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 122905600 _runtime‚ 1284.52888259 UHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚64681984 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 IBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚67 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚129891810770944 KDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚60 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 G>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4876 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 _step‚392 E8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚33815092 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 845178880 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚41348020 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 122905600 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚16096094 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚131046657617038 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 1157220083200 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚11031593 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚131815331210752 IDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚131815331210752 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚130658111127552 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚429 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚131046599229658 JBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚665 C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚4289 M@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚59049984 PCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚59049984 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚33819381 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 831258624 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚130658111127552 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 1157262028288 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4289 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚115157027248128 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 125829120 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚131815390260736 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚17723287 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚115156962566144 BDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 4654602240 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚130658128232448 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚4287 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚4026110 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 4517085184 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 5494538240 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚223 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚209 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 125829120 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚262 IBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚60 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚130658128232448 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17104896 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚44038144 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚262 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 4552982528 D;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚5264 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚113933296795648 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 4654602240 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚16091807 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚113933276151808 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚17723285 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17104896 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚322 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 1154846846094 F?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚60 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚4026175 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 5494538240 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 1157262028288 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 4552982528 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 5494538240 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚11031591 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚20643840 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 KBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚4876 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 1223686414336 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚61478912 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 1154805498074 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0ˆ^Êßù2î - _wandbruntime‚1284  _runtime‚12845àgÇ&ï ÁM REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚41948160 B6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚322 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚11033798 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚131073015595806 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚129917979525120 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 125829120 IBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚60 >5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5264 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚17726840 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 1157451151872 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚33821932 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚4026972 K>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59053056 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚15060770 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚262 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚113956344168448 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 125829120 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 1155036070686 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17104896 NADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚58387428 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 EDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚15060702 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 5494538240 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 5494538240 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚130684433465344 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 % _timestamp‚1.7866599361562252e+09 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 5368709120 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 1157493100032 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 1223978265600 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 1155077418754 NADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚59053056 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 831258624 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 _runtime‚1284.686535997 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚20643840 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 G@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚60 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚61478912 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚131073073983234 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚41348068 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚4293 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚262 _step‚393 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚131841884617216 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 5494538240 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚16095094 EDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521206 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚17726838 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 1223930036224 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 122306526 PDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚33826227 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚17726838 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17104896 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚130684450570240 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚41948160 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 122905600 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚322 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚16095094 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 4517085184 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 5368709120 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚4295 I@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚4876 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚129917996564480 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 5368709120 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 125829120 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654602240 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚4293 NGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚66 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚48229376 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 4654602240 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 4552982528 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚17726840 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚322 trainer/global_step‚4799 D5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5264 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚17726840 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 1157451151872 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚33821932 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚4026972 K>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59053056 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚15060770 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚262 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚113956344168448 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 125829120 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 1155036070686 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17104896 NADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚58387428 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 EDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚15060702 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 5494538240 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 5494538240 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚130684433465344 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 % _timestamp‚1.7866599361562252e+09 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 5368709120 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 1157493100032 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 1223978265600 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 1155077418754 NADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚59053056 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 831258624 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 _runtime‚1284.686535997 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚20643840 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 G@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚60 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚61478912 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚131073073983234 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚41348068 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚4293 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚262 _step‚393 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚131841884617216 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 5494538240 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚16095094 EDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521206 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚17726838 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 1223930036224 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 122306526 PDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚33826227 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚17726838 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17104896 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚130684450570240 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚41948160 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 122905600 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚322 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚16095094 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 4517085184 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 5368709120 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚4295 I@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚4876 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚129917996564480 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 5368709120 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 125829120 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654602240 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚4293 NGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚66 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚48229376 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 4654602240 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 4552982528 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚17726840 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚322 trainer/global_step‚4799 D6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚322 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚11033798 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚131073015595806 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚129917979525120 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 125829120 IBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚60‰ƺíÚ2ñ - _wandbruntime‚1284  _runtime‚12840ÖÈÞrò ì /loss/FRt1/detection_loss‚0.5500484293169805 $ _timestamp‚1.786659936157256e+09 0loss/FRt2/assignment_loss‚0.7595921672623247 /loss/FRt2/detection_loss‚0.4901471380777832 &loss/total_loss‚2.6230951120837127 epoch‚11 trainer/global_step‚4799 _runtime‚1284.687081958 _step‚394 0loss/FRt1/assignment_loss‚0.8233073774266244ÁÍzwó ñ 0loss/FRt2/assignment_loss‚0.7595921672623247 /loss/FRt2/detection_loss‚0.4901471380777832 &loss/total_loss‚2.6230951120837127 epoch‚11 trainer/global_step‚4799 _runtime‚1284.687081958 _step‚394 0loss/FRt1/assignment_loss‚0.8233073774266244 /loss/FRt1/detection_loss‚0.5500484293169805 $ _timestamp‚1.786659936157256e+09ŠJY¦Í2ô - _wandbruntime‚1289  _runtime‚1289îE0aõ Ø .Purity/*FRt/FRt_purity‚0.45312419533729553 3Purity/0FRt/event_proportion‚0.3301943838596344 /particle/accuracy_2_of_2‚0.7010456919670105 -Purity/2FRt/FRt_purity‚0.5313740372657776 *jet/accuracy_1_of_2‚0.5543525815010071 +jet/accuracy_2_of_2‚0.19175554811954498 0Purity/1FRt/event_purity‚0.44305405020713806 (particl(èv?}e/accuracy‚0.6281970739364624 % _timestamp‚1.7866599410935073e+09 epoch‚11 trainer/global_step‚4799 _step‚395 Purity/0FRt/event_purity‚1 .Purity/1FRt/FRt_purity‚0.44305405020713806 'particle/f_score‚0.5588628053665161 _runtime‚1289.623508578 /Purity/*FRt/event_purity‚0.4400549829006195 2Purity/*FRt/event_proportion‚0.669805645942688 +jet/accuracy_1_of_1‚0.29573512077331543 .particle/accuracy_1_of_2‚0.935930073261261 +particle/specificity‚0.6086947321891785 (loss/val_total_loss‚2.60382853925609 3Purity/1FRt/event_proportion‚0.6297432780265808 /Purity/2FRt/event_purity‚0.3916621804237366 particle/accuracy_1_of_1‚1 +particle/sensitivity‚0.6636273860931396 7validation_average_jet_accuracy‚0.30038484930992126 2Purity/2FRt/event_proportion‚0.040062315762043 +validation_accuracy‚0.19175554811954498_|«0ãö Ý epoch‚11 trainer/global_step‚4799 _step‚395 Purity/0FRt/event_purity‚1 .Purity/1FRt/FRt_purity‚0.44305405020713806 'particle/f_score‚0.5588628053665161 _runtime‚1289.623508578 /Purity/*FRt/event_purity‚0.4400549829006195 2Purity/*FRt/event_proportion‚0.669805645942688 +jet/accuracy_1_of_1‚0.29573512077331543 .particle/accuracy_1_of_2‚0.935930073261261 +particle/specificity‚0.6086947321891785 (loss/val_total_loss‚2.60382853925609 3Purity/1FRt/event_proportion‚0.6297432780265808 /Purity/2FRt/event_purity‚0.3916621804237366 particle/accuracy_1_of_1‚1 +particle/sensitivity‚0.6636273860931396 7validation_average_jet_accuracy‚0.30038484930992126 2Purity/2FRt/event_proportion‚0.040062315762043 +validation_accuracy‚0.19175554811954498 .Purity/*FRt/FRt_purity‚0.45312419533729553 3Purity/0FRt/event_proportion‚0.3301943838596344 /particle/accuracy_2_of_2‚0.7010456919670105 -Purity/2FRt/FRt_purity‚0.5313740372657776 *jet/accuracy_1_of_2‚0.5543525815010071 +jet/accuracy_2_of_2‚0.19175554811954498 0Purity/1FRt/event_purity‚0.44305405020713806 (particle/accuracy‚0.6281970739364624 % _timestamp‚1.7866599410935073e+09‹ÇÈl`“÷ ‚(:ˆ åˆùÓîñœá gpu.0.gpu‚4.0 gpu.0.memory‚0, gpu.0.memoryAllocated‚7.0313262939453125) gpu.0.memoryAllocatedBytes‚ 6039863296 gpu.0.temp‚32.0 gpu.0.powerWatts‚119.045' gpu.0.enforcedPowerLimitWatts‚500.0 gpu.0.powerPercent‚23.809 gpu.0.smClock‚1410 gpu.0.memoryClock‚1593$ gpu.0.correctedMemoryErrors‚1955# gpu.0.uncorrectedMemoryErrors‚0ü¦l³–ø ‚(:‹ åˆùÓŧèã disk.sda2.out‚0% disk./.usageGB‚466.44117736816406 disk.dm-1.in‚ 11683.3828125 disk.drbd2165.in‚0 disk.dm-0.in‚ 441.6796875# proc.memory.rssMB‚ 2330.78515625* proc.memory.percent‚14.225983619689941 proc.cpu.threads‚17 network.recv‚ 2.648552e+06 disk.drbd2165.out‚0 disk.sda3.in‚ 11683.3828125 disk.sda3.out‚ 26.06640625$ memory_percent‚65.92621803283691( proc.memory.availableMB‚ 5582.6484375 cpu‚26.969973871587975 disk.nvme0n1.in‚ 441.6796875# disk.dm-0.out‚30199.55126953125! disk.nvme0c0n1.in‚ 441.6796875( disk.nvme0c0n1.out‚30199.55126953125 disk.sda.in‚ 11683.3828125 disk.sda1.in‚0 network.sent‚ 6.442584e+06) disk./.usagePercent‚31.30325878339321& disk.nvme0n1.out‚30199.55126953125 disk.dm-1.out‚ 26.06640625 disk.sda.out‚ 26.06640625 disk.sda1.out‚0 disk.sda2.in‚0±b‹í2ù - _wandbruntime‚1302  _runtime‚1302àsPÄú ¾ % lr-AdamW/pg1‚0.001317248514009313 % lr-AdamW/pg2‚0.001317248514009313 trainer/global_step‚4849 $ _timestamp‚1.786659954345346e+09 _runtime‚1302.875400097 _step‚396`¾ÞÉû à % lr-AdamW/pg1‚0.001317248514009313 % lr-AdamW/pg2‚0.001317248514009313 trainer/global_step‚4849 $ _timestamp‚1.786659954345346e+09 _runtime‚1302.875400097 _step‚396Œ-û 22ü - _wandbruntime‚1302  _runtime‚1302öyŸ"½'ý ·O XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 5368709120 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17104896 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 1172916520448 KBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚4876 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚116625029562368 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚20643840 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 1170433280788 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚223 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚115384935743488 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 5368709120 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 122306526 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚133538039807488 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚133538098857472 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 F?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚60 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚17961541 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 125829120 trainer/global_step‚4849 IDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 125829120 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚17961541 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 1240093818880 F>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚132365165232128 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚429 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚116624966977536 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚262 @7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚5264 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 125829120 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚322 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 1170474628808 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 5368709120 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚132365182337024 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚4075905 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚322 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚17961543 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚322 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 4654602240 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 1172874575360 IDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4876 PCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚58387380 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚11182457 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚132752474568392 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚262 _runtime‚1302.879910787 M@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚59049984 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 4654602240 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4289 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 4628521206 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚4287 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚4075836 UHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚62584832 E8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚34240642 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚17961543 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚132365182337024 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚41945088 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚16283388 PIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚69 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 5494538240O˶cÂ'þ ¼O PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚16283388 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚4287 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚15258362 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚61478912 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚132752416181012 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚131581982900224 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚41348020 IBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚60 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 4552982528 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 1172916520448 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚34244931 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 IBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚71 G>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4876 PCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚58387380 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚11182457 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚132752474568392 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚262 _runtime‚1302.879910787 M@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚59049984 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 4654602240 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4289 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 4628521206 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚4287 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚4075836 UHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚62584832 E8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚34240642 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚17961543 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚132365182337024 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚41945088 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚16283388 PIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚69 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 5494538240 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 5368709120 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17104896 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 1172916520448 KBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚4876 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚116625029562368 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚20643840 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 1170433280788 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚223 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚115384935743488 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 5368709120 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 122306526 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚133538039807488 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚133538098857472 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 F?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚60 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚17961541 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 125829120 trainer/global_step‚4849 IDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 125829120 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚17961541 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 1240093818880 F>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚132365165232128 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚429 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚116624966977536 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚262 @7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚5264 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 125829120 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚322 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 1170474628808 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 5368709120 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚132365182337024 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚4075905 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚322 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚17961543 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚322 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 4654602240 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 1172874575360 I6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚322 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 >5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5264 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚132391504674816 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚34251777 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚20643840 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚4295 NADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚59053056 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17104896 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 1173147592192 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 122905600 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚17965096 _runtime‚1303.045279327 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚11184662 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17104896 B9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚5264 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚322 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 4552982528 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 K>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59053056 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521206 D=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚60 % _timestamp‚1.7866599545149424e+09 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚133564652267008 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚41948160 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚131608185733120 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚262 G@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚60 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 _step‚398 E6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚322 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚16286681 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚133564593213952 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 4552982528 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚115407983116288 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚4076734 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 831258624 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚41348068 DDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚15261328 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚17965096 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚132391504674816 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 845178880 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 125829120 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚132391487569920 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚34247482 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚34247482 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 4654602240 E=DeviceStatsMonitor.on_train_batch_enÝéFd/inactive_split.all.peak‚429 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚115407962472448 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 1173105644032 BDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654602240 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚116648257087488 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 5368709120 PDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚15261328 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚17965096 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚132391504674816 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 845178880 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 125829120 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚132391487569920 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚34247482 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚34247482 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 4654602240 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚429 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚115407962472448 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 1173105644032 BDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654602240 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚116648257087488 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 5368709120 P6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚322 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 >5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5264 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚132391504674816 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚34251777 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚20643840 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚4295 NADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚59053056 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17104896 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 1173147592192 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 122905600 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚17965096 _runtime‚1303.045279327 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚11184662 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17104896 B9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚5264 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚322 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 4552982528 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 K>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59053056 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521206 D=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚60 % _timestamp‚1.7866599545149424e+09 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚133564652267008 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚41948160 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚131608185733120 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚262 G@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚60 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 _step‚398 E6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚322 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚16286681 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚133564593213952 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 4552982528 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚115407983116288Ž&†ùT2‚ - _wandbruntime‚1303  _runtime‚1303°BYÚtƒ î trainer/global_step‚4849 0loss/FRt1/assignment_loss‚0.6887166543865004 /loss/FRt1/detection_loss‚0.5172496306309832 0loss/FRt2/detection_loss‚0.48280942104548413 epoch‚12 % _timestamp‚1.7866599545160391e+09 _runtime‚1303.045840446 _step‚399 0loss/FRt2/assignment_loss‚0.6080854053859882 &loss/total_loss‚2.2968611114489557¯Tù·y„ ó /loss/FRt1/detection_loss‚0.5172496306309832 0loss/FRt2/detection_loss‚0.48280942104548413 epoch‚12 % _timestamp‚1.7866599545160391e+09 _runtime‚1303.045840446 _step‚399 0loss/FRt2/assignment_loss‚0.6080854053859882 &loss/total_loss‚2.2968611114489557 trainer/global_step‚4849 0loss/FRt1/assignment_loss‚0.6887166543865004C*R# … ‚(:• ôˆùÓž×õà gpu.0.gpu‚63.0 gpu.0.memory‚12, gpu.0.memoryAllocated‚7.0313262939453125) gpu.0.memoryAllocatedBytes‚ 6039863296 gpu.0.temp‚35.0 gpu.0.powerWatts‚206.82' gpu.0.enforcedPowerLimitWatts‚500.0) gpu.0.powerPercent‚41.364000000000004 gpu.0.smClock‚1410 gpu.0.memoryClock‚1593$ gpu.0.correctedMemoryErrors‚1955# gpu.0.uncorrectedMemoryErrors‚0“Nº±Ž† ‚(:ƒ ôˆùÓÉëºã% disk.nvme0n1.out‚30235.0107421875( proc.memory.availableMB‚ 5582.5078125* proc.memory.percent‚14.225983619689941 cpu‚26.97106972779984 disk.sda.out‚ 26.296875 disk.sda1.out‚0 disk.nvme0n1.in‚ 452.5390625" disk.dm-0.out‚30235.0107421875 disk.sda3.in‚11722.57421875$ memory_percent‚65.92707633972168# proc.memory.rssMB‚ 2330.78515625 disk.dm-1.in‚ 11721.6484375 disk.dm-1.out‚ 26.296875 disk.drbd2165.out‚0 disk.sda3.out‚ 26.296875 proc.cpu.threads‚17 network.sent‚ 6.514337e+06 network.recv‚ 2.676161e+06$ disk./.usageGB‚465.3520851135254' disk.nvme0c0n1.out‚30235.0107421875 disk.sda2.in‚0 disk.sda2.out‚0 disk.drbd2165.in‚0 disk.dm-0.in‚ 452.5390625* disk./.usagePercent‚31.230168888375143! disk.nvme0c0n1.in‚ 452.5390625 disk.sda.in‚11722.57421875 disk.sda1.in‚0…•Ó`2‡ - _wandbruntime‚1315  _runtime‚1315,ýüLj Á & lr-AdamW/pg1‚0.0013132982071347248 & lr-AdamW/pg2‚0.0013132982071347248 trainer/global_step‚4899 % _timestamp‚1.7866599672702749e+09 _runtime‚1315.800349695 _step‚400œûËÛ̉ Æ & lr-AdamW/pg1‚0.0013132982071347248 & lr-AdamW/pg2‚0.0013132982071347248 trainer/global_step‚4899 % _timestamp‚1.7866599672702749e+09 _runtime‚1315.800349695 _step‚400aŸ2Š - _wandbruntime‚1315  _runtime‚1315Û™½'‹ ·O YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚133681282121728 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚34587683 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚16448490 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚4287 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 5368709120 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 122905600 KBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚4876 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 4517085184 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚262 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 1184503286272 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚11292705 IDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚134865743462912 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 831258624 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 122306526 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 4654602240 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚18139191 PCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚58387380 C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚4289 E8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚34583394 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚16444203 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 122905600 IBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚73 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚15409564 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 845178880 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚322 KDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚60 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 1252341986816 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 IBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚60 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚134865802512896 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 4552982528 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚132891272577024 _runtime‚1315.804927222 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 5368709120 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 F?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚60 M@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚59049984 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚429 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚117789625726464 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 PCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚59049984 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 5494538240 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚322 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚262 trainer/global_step‚4899 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚41945088 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚322 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚41348020 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 1181995091396 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 125829120 D;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚5264 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚20643840 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 1252388122112 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚4287 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 4654602240 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 1184461341184 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚262 F>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚134865743462912 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚133681299226624 JBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚665 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17104896 IDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4876 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4289 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 5494538240 _step‚401 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚18139193 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17104896 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 4628521206 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 125829120 @7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚5264 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚61478912 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚134073267668420 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚18139193 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚134073326055800 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚132891289616384 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 1182036439416 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 125829120 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚133681282121728 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚116537283739648 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚223 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚209 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 BDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚262 F>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚134865743462912 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚133681299226624 JBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚665 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17104896 IDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4876 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4289 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 5494538240 _step‚401 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚18139193 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17104896 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 4628521206 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 125829120 @7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚5264 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚61478912 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚134073267668420 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚18139193 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚134073326055800 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚132891289616384 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 1182036439416 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 125829120 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚133681282121728 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚116537283739648 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚223 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚209 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 BDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚134865743462912 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 831258624 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 122306526‘ü õ2 - _wandbruntime‚1315  _runtime‚13154ðwFÇ&Ž ÁM _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚116560331112448 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚46132224 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654602240 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚34594529 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚223 NGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚62 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚132917458370560 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 5494538240 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 125829120 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚262 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 4552982528 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17104896 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 122905600 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 122306526 trainer/global_step‚4899 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚34594529 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚134892355922432 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 1184692409856 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚4117738 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚4117676 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 5494538240 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚15412586 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 1182225664008 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚133707604459520 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚15412650 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 1182267012076 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚134892296869376 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚133707621564416 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 831258624 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚41948160 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚116560351756288 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚132917475409920 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 125829120 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚209 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚322 G@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚60 EDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521206 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 5368709120 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚117812984906240 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚41948160 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚20643840 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 5494538240 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚34590234 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚134099742421996 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚134099684034568 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚322 K>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59053056 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 122905600 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚16451783 NADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚58387428 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 125829120 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 5368709120 E6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚322 % _timestamp‚1.7866599674476595e+09 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17104896 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 1184692409856 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚18142744 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 845178880 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚18142744 I@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚4876 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 1252587017728 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 5368709120 NADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚59053056 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚16451783 G@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚64 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚34590234 D5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5264 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 1184734358016 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚133707604459520 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 SFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚66776064!.´3Ì& ÆM EDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521206 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 5368709120 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚117812984906240 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚41948160 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚20643840 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 5494538240 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚34590234 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚134099742421996 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚134099684034568 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚322 K>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59053056 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 122905600 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚16451783 NADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚58387428 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 125829120 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 5368709120 E6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚322 % _timestamp‚1.7866599674476595e+09 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17104896 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 1184692409856 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚18142744 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 845178880 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚18142744 I@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚4876 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 1252587017728 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 5368709120 NADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚59053056 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚16451783 G@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚64 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚34590234 D5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5264 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 1184734358016 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚133707604459520 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 SFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚66776064 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚116560331112448 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚46132224 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654602240 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚34594529 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚223 NGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚62 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚132917458370560 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 5494538240 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 125829120 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚262 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 4552982528 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17104896 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 122905600 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 122306526 trainer/global_step‚4899 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚34594529 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚134892355922432 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 1184692409856 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚4117738 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚4117676 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 5494538240 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚15412586 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 1182225664008 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚133707604459520 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚15412650 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 1182267012076 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚134892296869376 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚133707621564416 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 831258624 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚41948160 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚116560351756288 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚132917475409920 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 125829120 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚209 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚322 G@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚60’ˆâ2 - _wandbruntime‚1315  _runtime‚1315­ŒŽHr‘ ì 0loss/FRt1/assignment_loss‚0.8731588374419116 0loss/FRt2/assignment_loss‚0.7523272961919378 %loss/total_loss‚2.718639503324328 epoch‚12 trainer/global_step‚4899 % _timestamp‚1.7866599674494336e+09 _step‚403 /loss/FRt1/detection_loss‚0.5619990034387551 /loss/FRt2/detection_loss‚0.5311543662517235 _runtime‚1315.979237984EšÅ†w’ ñ 0loss/FRt1/assignment_loss‚0.8731588374419116 0loss/FRt2/assignment_loss‚0.7523272961919378 %loss/total_loss‚2.718639503324328 epoch‚12 trainer/global_step‚4899 % _timestamp‚1.7866599674494336e+09 _step‚403 /loss/FRt1/detection_loss‚0.5619990034387551 /loss/FRt2/detection_loss‚0.5311543662517235 _runtime‚1315.979237984“P@ƒÒ “ ‚(:• ƒ‰ùÓ÷†¦â gpu.0.gpu‚27.0 gpu.0.memory‚3, gpu.0.memoryAllocated‚7.0313262939453125) gpu.0.memoryAllocatedBytes‚ 6039863296 gpu.0.temp‚35.0 gpu.0.powerWatts‚194.952' gpu.0.enforcedPowerLimitWatts‚500.0) gpu.0.powerPercent‚38.990399999999994 gpu.0.smClock‚1410 gpu.0.memoryClock‚1593$ gpu.0.correctedMemoryErrors‚1955# gpu.0.uncorrectedMemoryErrors‚0éE’쎔 ‚(:ƒ ƒ‰ùÓÀú¤ä network.recv‚ 2.704053e+06* disk./.usagePercent‚31.498616084711706 disk.dm-1.out‚ 26.6953125 disk.dm-0.in‚ 453.9140625 disk.sda.in‚12490.59765625 disk.sda.out‚ 26.6953125 disk.sda2.out‚0& proc.memory.availableMB‚ 5582.71875 disk.dm-1.in‚12490.59765625 disk.drbd2165.in‚0 disk.sda3.in‚12490.59765625! disk.nvme0c0n1.in‚ 453.9140625# proc.memory.rssMB‚ 2330.78515625* proc.memory.percent‚14.225983619689941$ disk./.usageGB‚469.3521423339844 disk.sda3.out‚ 26.6953125& disk.nvme0c0n1.out‚31861.900390625 disk.nvme0n1.in‚ 453.9140625$ disk.nvme0n1.out‚31861.900390625$ memory_percent‚65.92578887939453 cpu‚26.970618461008435 network.sent‚ 6.584916e+06 disk.drbd2165.out‚0! disk.dm-0.out‚31812.150390625 disk.sda1.in‚0 disk.sda1.out‚0 disk.sda2.in‚0 proc.cpu.threads‚17 ]ߢ2• - _wandbruntime‚1331  _runtime‚1331Öõ9bÆ– À $ _timestamp‚1.786659982810799e+09 _runtime‚1331.340911129 _step‚404 & lr-AdamW/pg1‚0.0013093117206689007 & lr-AdamW/pg2‚0.0013093117206689007 trainer/global_step‚4949’d=Ë— Å & lr-AdamW/pg1‚0.0013093117206689007 & lr-AdamW/pg2‚0.0013093117206689007 trainer/global_step‚4949 $ _timestamp‚1.786659982810799e+09 _runtime‚1331.340911129 _step‚404”¢©“Ü2˜ - _wandbruntime‚1331  _runtime‚1331×\“k ™ ¶O BDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4876 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚18316843 C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚4289 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17104896 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚135394119155828 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚136193506168320 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚34930435 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚18316841 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚11402955 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 5494538240 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚262 D;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚5264 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚134200562253824 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚11402957 E8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚34926146 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚4287 IDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚20643840 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 5368709120 IDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚136193447118336 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚118954267556352 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 1264657117696 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚322 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 122306526 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 5368709120 KDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚60 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 PCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚58387380 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 125829120 JBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚665 UHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚62584832 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 4517085184 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚134997416116224 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 1196090052096 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚117689652379648 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 1196090052096 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚41945088 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚136193506168320 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚41945088 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4289 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚429 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚322 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚262 F?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚60 trainer/global_step‚4949 _runtime‚1331.342453231 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚134997399011328 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 845178880 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 4628521206 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚18316843 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 125829120 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 4654602240 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 5368709120 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 122905600 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚15560634 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 1193556902004 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚322 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚136193447118336 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17104896 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 4552982528 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚4157677 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚16609305 PIDeviceStatsMonitor.onLÔÅ-Q_train_batch_start/inactive_split.small_pool.current‚63 $ _timestamp‚1.786659982812115e+09 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 122905600 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚118954330141184 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 1193598250024 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚16613592 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚135394177543208 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 5494538240 IBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚60 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 125829120 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚15560569 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚4157614 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚41940992 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 1264615176704 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚322 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 5494538240 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚223 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚18316841 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚16613592ÀVÅSÁ'š »O ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 1196090052096 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚41945088 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚136193506168320 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚41945088 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4289 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚429 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚322 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚262 F?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚60 trainer/global_step‚4949 _runtime‚1331.342453231 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚134997399011328 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 845178880 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 4628521206 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚18316843 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 125829120 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 4654602240 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 5368709120 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 122905600 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚15560634 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 1193556902004 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚322 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚136193447118336 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17104896 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 4552982528 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚4157677 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚16609305 PIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚63 $ _timestamp‚1.786659982812115e+09 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 122905600 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚118954330141184 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 1193598250024 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚16613592 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚135394177543208 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 5494538240 IBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚60 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 125829120 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚15560569 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚4157614 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚41940992 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 1264615176704 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚322 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 5494538240 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚223 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚18316841 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚16613592 BDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4876 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚18316843 C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚4289 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17104896 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚135394119155828 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚136193506168320 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚34930435 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚18316841 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚11402955 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 5494538240 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚262 D;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚5264 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚134200562253824 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚11402957 E8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚34926146 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚4287 IDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚20643840 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 5368709120 IDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚136193447118336 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚118954267556352 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 1264657117696 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚322 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 122306526 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 5368709120 KDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚60 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 PCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚58387380 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 125829120 JBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚665 UHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚62584832 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 4517085184 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚134997416116224 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 1196090052096 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚117689652379648 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1•ÖÞ –2› - _wandbruntime‚1331  _runtime‚1331ÓqÀ8Ç&œ ÁM C6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚34932986 NGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚67 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 125829120 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 125829120 _step‚406 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚429 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚16616885 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚41937920 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 BDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 trainer/global_step‚4949 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚135420593909404 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚134226765086720 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 122306526 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚16612592 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚136220059577856 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 4552982528 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 1196279175680 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654602240 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚135023738454016 SFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚62581760 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚117712720396288 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚135420535521976 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 1193828822684 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 125829120 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 D=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚60 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚135023721349120 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚322 K>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59053056 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚18320394 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚16616885 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚117712699752448 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚61478912 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 EDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚15563604 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 IBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚60 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚11405160 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚4295 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚4293 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚4293 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚134226748047360 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17104896 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17104896 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚41948160 G@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚69 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 845178880 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚135023721349120 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 5368709120 B9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚5264 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 1264901424640 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚18320394 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚209 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521206 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚34937281 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚16612592 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚118977559239168 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 831258624 NADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚58387428 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚41348068 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚18320396 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 1196279175680 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚34937281 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚322 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 5494538240 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 4552982528 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 1196321123840 NADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚59053056 I@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚4876 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚4158444 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚15563673 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 1193787474616 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚262 G@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚60 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚136220000524800 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 122905600 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚223 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚118977621820928 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 4517085184 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 5494538240 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 5494538240 P8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚11405162 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚322 >5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5264 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 5368709120 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 5368709120Hò²Ì& ÆM NADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚58387428 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚41348068 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚18320396 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 1196279175680 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚34937281 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚322 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 5494538240 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 4552982528 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 1196321123840 NADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚59053056 I@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚4876 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚4158444 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚15563673 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 1193787474616 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚262 G@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚60 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚136220000524800 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 122905600 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚223 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚118977621820928 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 4517085184 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 5494538240 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 5494538240 P8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚11405162 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚322 >5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5264 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 5368709120 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 5368709120 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚34932986 NGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚67 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 125829120 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 125829120 _step‚406 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚429 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚16616885 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚41937920 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 BDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 trainer/global_step‚4949 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚135420593909404 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚134226765086720 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 122306526 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚16612592 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚136220059577856 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 4552982528 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 1196279175680 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654602240 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚135023738454016 SFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚62581760 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚117712720396288 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚135420535521976 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 1193828822684 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 125829120 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 D=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚60 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚135023721349120 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚322 K>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59053056 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚18320394 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚16616885 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚117712699752448 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚61478912 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 EDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚15563604 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 IBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚60 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚11405160 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚4295 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚4293 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚4293 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚134226748047360 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17104896 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17104896 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚41948160 G@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚69 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 845178880 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚135023721349120 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 5368709120 B9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚5264 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 1264901424640 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚18320394 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚209 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521206 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚34937281 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚16612592 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚118977559239168 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 831258624–JG¡I2ž - _wandbruntime‚1331  _runtime‚1331tÏË)sŸ í trainer/global_step‚4949 % _timestamp‚1.7866599830304365e+09 _runtime‚1331.560291749 _step‚407 0loss/FRt2/assignment_loss‚0.6643463584376443 &loss/total_loss‚2.3893878930325236 0loss/FRt1/assignment_loss‚0.7283381538075484 /loss/FRt1/detection_loss‚0.5341818059029869 /loss/FRt2/detection_loss‚0.4625215748843439 epoch‚12"6L  ò 0loss/FRt1/assignment_loss‚0.7283381538075484 /loss/FRt1/detection_loss‚0.5341818059029869 /loss/FRt2/detection_loss‚0.4625215748843439 epoch‚12 trainer/global_step‚4949 % _timestamp‚1.7866599830304365e+09 _runtime‚1331.560291749 _step‚407 0loss/FRt2/assignment_loss‚0.6643463584376443 &¼,&loss/total_loss‚2.3893878930325236— Õä•¡ ‚(:Š ’‰ùÓÙêðà gpu.0.gpu‚28.0 gpu.0.memory‚6, gpu.0.memoryAllocated‚7.0313262939453125) gpu.0.memoryAllocatedBytes‚ 6039863296 gpu.0.temp‚34.0 gpu.0.powerWatts‚215.488' gpu.0.enforcedPowerLimitWatts‚500.0 gpu.0.powerPercent‚43.0976 gpu.0.smClock‚1410 gpu.0.memoryClock‚1593$ gpu.0.correctedMemoryErrors‚1955# gpu.0.uncorrectedMemoryErrors‚0Tò py¢ ‚(:î ’‰ùÓйã disk.sda3.out‚ 26.8671875 disk.sda.out‚ 26.8671875 disk.sda1.in‚0 disk.sda2.out‚0 disk.nvme0n1.in‚462.125& disk.nvme0n1.out‚42897.62158203125 disk.dm-1.out‚ 26.8671875 network.recv‚ 2.731165e+06$ disk./.usageGB‚481.3186798095703( disk.nvme0c0n1.out‚42897.62158203125 disk.sda.in‚ 12917.8515625 disk.sda1.out‚0' proc.memory.availableMB‚ 5660.296875) disk./.usagePercent‚32.30170045529199 disk.sda3.in‚ 12917.8515625 disk.sda2.in‚0 disk.dm-1.in‚ 12917.8515625$ memory_percent‚65.45228958129883 proc.cpu.threads‚17 disk.nvme0c0n1.in‚462.125 disk.drbd2165.in‚0 disk.drbd2165.out‚0 disk.dm-0.in‚462.125# disk.dm-0.out‚42897.62158203125 proc.memory.rssMB‚2253' proc.memory.percent‚13.751220703125 cpu‚26.969698669108105 network.sent‚ 6.655269e+06Ñîz*2£ - _wandbruntime‚1345  _runtime‚1345ì“çøŤ ¿ % lr-AdamW/pg2‚0.001305289310656417 trainer/global_step‚4999 % _timestamp‚1.7866599967066357e+09 _runtime‚1345.236801644 _step‚408 % lr-AdamW/pg1‚0.00130528931065641750FBÊ¥ Ä % lr-AdamW/pg1‚0.001305289310656417 % lr-AdamW/pg2‚0.001305289310656417 trainer/global_step‚4999 % _timestamp‚1.7866599967066357e+09 _runtime‚1345.236801644 _step‚408˜MwÑõ2¦ - _wandbruntime‚1345  _runtime‚1345{Žž°½'§ ·O ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚120118927858688 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚41945088 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚136714970643236 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚322 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 1205160060632 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 4654602240 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚4287 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚15712082 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 5368709120 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 125829120 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 125829120 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚16778694 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 4552982528 G>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4876 E8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚35268898 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚4287 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚4198875 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚118842041663488 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚18494493 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 5494538240 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚4289 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 122905600 BDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17104896 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚15712011 @7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚5264 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚16778694 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 4654602240 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚41348020 % _timestamp‚1.7866599967077866e+09 KBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚4876 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 F>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚209 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 1205118712612 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 5494538240 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚262 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚137521150773760 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4289 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚118842021019648 trainer/global_step‚4999 IDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4876 E8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚35268898 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚4287 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚4198875 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚118842041663488 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚18494493 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 5494538240 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚4289 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 122905600 BDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17104896 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚15712011 @7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚5264 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚16778694 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 4654602240 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚41348020 % _timestamp‚1.7866599967077866e+09 KBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚4876 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 F>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚209 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 1205118712612 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 5494538240 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚262 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚137521150773760 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4289 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚118842021019648 trainer/global_step‚4999 IÚÇ&ª ÁM J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚18498044 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚136339838238720 NADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚59053056 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 NADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚58387428 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 5494538240 IBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚60 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚136339855343616 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚4293 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚429 SFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚68873216 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 831258624 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚48229376 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 4552982528 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521206 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 5494538240 G@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚60 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚16777694 EDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654602240 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚136741445396812 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚41348068 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚16777694 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 1207865941504 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚15715024 >5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5264 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚35280033 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚120142217444352 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 4552982528 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚118865089036288 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 _step‚410 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚137547704180224 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚135536054763520 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚262 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 122905600 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚136339838238720 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚18498046 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 D=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚60 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚35280033 H@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚665 G@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚65 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 125829120 B9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚5264 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚11515412 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚20643840 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚41948160 I@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚4876 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚61478912 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 1205349285224 D6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚322 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚16781987 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚41948160 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚135536037724160 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚322 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚322 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 1207907889664 NGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚63 BDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚262 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚4199614 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚16781987 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚18498044 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 1207907889664 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚4199677 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 _runtime‚1345.445615664 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚137547763233280 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 4654602240 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚15715089 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17104896 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17104896 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 122905600 EDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59053056 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 5368709120 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚262 trainer/global_step‚4999 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 1207865941504ÐnÚ£« ÆM JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚16781987 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚18498044 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 1207907889664 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚4199677 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 _runtime‚1345.443.cøÅ$5615664 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚137547763233280 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 4654602240 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚15715089 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17104896 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17104896 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 122905600 EDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59053056 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 5368709120 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚262 trainer/global_step‚4999 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 1207865941504 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚18498044 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚136339838238720 NADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚59053056 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 NADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚58387428 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 5494538240 IBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚60 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚136339855343616 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚4293 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚429 SFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚68873216 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 831258624 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚48229376 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 4552982528 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521206 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 5494538240 G@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚60 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚16777694 EDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654602240 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚136741445396812 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚41348068 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚16777694 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 1207865941504 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚15715024 >5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5264 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚35280033 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚120142217444352 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 4552982528 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚118865089036288 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 _step‚410 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚137547704180224 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚135536054763520 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚262 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 122905600 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚136339838238720 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚18498046 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 D=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚60 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚35280033 H@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚665 G@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚65 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 125829120 B9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚5264 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚11515412 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚20643840 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚41948160 I@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚4876 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚61478912 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 1205349285224 D6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚322 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚16781987 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚41948160 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚135536037724160 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚322 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚322 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 1207907889664 NGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚63 BDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚262 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚4199614 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0š4B÷‘2¬ - _wandbruntime‚1345  _runtime‚1345—pÂ#r­ ì /loss/FRt2/detection_loss‚0.4506712252079459 &loss/total_loss‚2.4436020067594013 trainer/global_step‚4999 _runtime‚1345.446229785 _step‚411 0loss/FRt1/assignment_loss‚0.7553108963577347 0loss/FRt2/assignment_loss‚0.6749383959868249 .loss/FRt1/detection_loss‚0.562681489206896 epoch‚12 % _timestamp‚1.7866599969163735e+09£êÁw® ñ 0loss/FRt1/assignment_loss‚0.7553108963577347 0loss/FRt2/assignment_loss‚0.6749383959868249 .loss/FRt1/detection_loss‚0.562681489206896 epoch‚12 % _timestamp‚1.7866599969163735e+09 /loss/FRt2/detection_loss‚0.4506712252079459 &loss/total_loss‚2.4436020067594013 trainer/global_step‚4999 _runtime‚1345.446229785 _step‚411›qp\I”¯ ‚(:‰ ¡‰ùÓúÖ×â gpu.0.gpu‚0.0 gpu.0.memory‚0, gpu.0.memoryAllocated‚7.0313262939453125) gpu.0.memoryAllocatedBytes‚ 6039863296 gpu.0.temp‚34.0 gpu.0.powerWatts‚166.379' gpu.0.enforcedPowerLimitWatts‚500.0 gpu.0.powerPercent‚33.2758 gpu.0.smClock‚1410 gpu.0.memoryClock‚1593$ gpu.0.correctedMemoryErrors‚1955# gpu.0.uncorrectedMemoryErrors‚0>r4æ„° ‚(:ù ¡‰ùÓ×Àýã) proc.memory.percent‚13.75124454498291 proc.cpu.threads‚17 disk.nvme0c0n1.in‚462.125 disk.sda.out‚ 27.27734375 disk.sda1.out‚0 disk.dm-1.in‚ 12917.8515625 disk.dm-1.out‚ 27.27734375) proc.memory.availableMB‚ 5660.27734375) disk./.usagePercent‚32.83803444747266 disk.sda3.out‚ 27.27734375 disk.sda1.in‚0 disk.sda2.in‚0 disk.drbd2165.in‚0" disk.dm-0.out‚49371.2314453125 cpu‚26.97000449953707 network.sent‚ 6.719006e+06$ disk./.usageGB‚489.3104438781738' disk.nvme0c0n1.out‚49449.9814453125 disk.sda2.out‚0 disk.nvme0n1.in‚462.125 disk.drbd2165.out‚0 disk.dm-0.in‚462.125$ memory_percent‚65.45240879058838 network.recv‚ 2.75223e+06 disk.sda3.in‚ 12917.8515625 disk.sda.in‚ 12917.8515625% disk.nvme0n1.out‚49449.9814453125# proc.memory.rssMB‚ 2253.00390625Ô©E+2± - _wandbruntime‚1359  _runtime‚1359”ic¢Å² ¿ % lr-AdamW/pg2‚0.001301231235449153 trainer/global_step‚5049 % _timestamp‚1.7866600110177572e+09 _runtime‚1359.548849445 _step‚412 % lr-AdamW/pg1‚0.001301231235449153 撎ʳ Ä trainer/global_step‚5049 % _timestamp‚1.7866600110177572e+09 _runtime‚1359.548849445 _step‚412 % lr-AdamW/pg1‚0.001301231235449153 % lr-AdamW/pg2‚0.001301231235449153œH0îô2´ - _wandbruntime‚1359  _runtime‚1359pU¯¼'µ ¶O WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚41945088 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 5494538240 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚137629632790528 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 1219221638656 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚15863110 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 IBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚77 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚119994389659648 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚35615939 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚4239655 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚20643840 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 125829120 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚262 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 1289176759296 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 F?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚60 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚138848913479168 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 125829120 M@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚59049984 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 4552982528 D;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚5264 JBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚665 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 831258624 trainer/global_step‚5049 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚138848854429184 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚137629649895424 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚18672141 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 1216721871240 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚18672143 UHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚66779136 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚119994410303488 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚138035880518024 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚322 PCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚59049984 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚138848913479168 KBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚4876 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚15863187 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚121283633198080 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚4287 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 122306526 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚138848854429184 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚137629649895424 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 4552982528 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚46135296 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚41348020 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 5494538240 PIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚75 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚262 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 845178880 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 G>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4876 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 122905600 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚11623457 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚223 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚61478912 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚322 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 4628521206 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚16939509 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 4654602240 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚16943796 F>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 IBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚60 $ _timestamp‚1.786660011018749e+09 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚121283566418944 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚136819141607424 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 1216680523220 _runtime‚1359.549186942 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 IDeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4289 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚18672143 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 5494538240 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚4287 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚136819158646784 _step‚413 IDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 F?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚60 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚138848913479168 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 125829120 M@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚59049984 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 4552982528 D;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚5264 JBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚665 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 831258624 trainer/global_step‚5049 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚138848854429184 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚137629649895424 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚18672141 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 1216721871240 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚18672143 UHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚66779136 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚119994410303488 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚138035880518024 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚322 PCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚59049984 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚138848913479168 KBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚4876 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚15863187 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚121283633198080 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚4287 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 122306526 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚138848854429184 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚137629649895424 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 4552982528 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚46135296 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚41348020 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 5494538240 PIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚75 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚262 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 845178880 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 G>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4876 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 122905600 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚11623457 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚223 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚61478912 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚322 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 4628521206 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚16939509 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 4654602240 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚16943796 F>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 IBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚60 $ _timestamp‚1.786660011018749e+09 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚121283566418944 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚136819141607424 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 1216680523220 _runtime‚1359.549186942 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 IDeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4289 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚18672143 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 5494538240 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚4287 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚136819158646784 _step‚413 I5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5264 K>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59053056 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚322 S?DeviceStatsMonitor.on_train_batch_end/requestËIh´Ã%ed_bytes.all.freed‚138062238496792 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚322 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 4552982528 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 122905600 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 125829120 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚137655972233216 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 122306526 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 5368709120 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚138875466888704 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚18675696 trainer/global_step‚5049 PDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚15866134 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚209 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 1216911095832 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚262 NGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚52 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚121306923829248 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚39840768 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚322 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17104896 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚11625660 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 NADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚58387428 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 5368709120 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚41348068 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚15866188 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚16942796 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚138875466888704 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚137655972233216 B9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚5264 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚18675694 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚137655955128320 H@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚665 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚20643840 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 5494538240 _step‚414 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 IBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚60 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 125829120 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚322 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚4240526 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚262 % _timestamp‚1.7866600111949568e+09 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚4293 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚120017437032448 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 4517085184 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚262 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 831258624 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚138875407835648 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654602240 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚35618490 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 4654602240 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚137655955128320 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 1219452707328 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚35622785 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 G@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚54 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚121306863344640 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 845178880 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚61478912 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 _runtime‚1359.725299471 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 122905600 I@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚4876 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚223 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 5368709120 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 1219452707328 DDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521206 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 125829120 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 B8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 G@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚60 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚4295 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚4293 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 1289466152960 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚138062296884220 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚136845344440320 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 1216952443900 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 5494538240 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚41948160 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚429 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 NADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚59053056 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚35622785 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚16947089 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚11625662 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17104896 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 1219494655488 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚4240474 E5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5264 K>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59053056 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚322 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚138062238496792 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚322 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 4552982528 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 122905600 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 125829120 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚137655972233216 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 122306526 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 5368709120 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚138875466888704 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚18675696 trainer/global_step‚5049 PDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚15866134 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚209 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 1216911095832 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚262 NGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚52 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚121306923829248 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚39840768 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚322 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17104896 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚11625660 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 NADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚58387428 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 5368709120 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚41348068 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚15866188 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚16942796 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚138875466888704 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚137655972233216 B9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚5264 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚18675694 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚137655955128320 H@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚665 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚20643840 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 5494538240 _step‚414 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 IBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚60 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 125829120 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚322 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚4240526 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚262 % _timestamp‚1.7866600111949568e+09 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚4293 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚120017437032448 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 4517085184 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚262 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 831258624 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚138875407835648 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654602240 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚35618490 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 4654602240 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚137655955128320 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 1219452707328 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚35622785 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 G@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚54 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚121306863344640 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 845178880 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚61478912 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 _runtime‚1359.725299471 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 122905600 I@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚4876 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚223 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 5368709120 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 1219452707328 DDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521206 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 125829120 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 B8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 G@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚60 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚4295 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚4293 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 1289466152960 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚138062296884220 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚136845344440320 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 1216952443900 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 5494538240 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚41948160 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚429 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0ž¾³;À2º - _wandbruntime‚1359  _runtime‚1359xð*q» ë 0loss/FRt2/assignment_loss‚0.7207613822968976 epoch‚12 % _timestamp‚1.7866600111961062e+09 _runtime‚ 1359.72602405 0loss/FRt1/assignment_loss‚0.7617371977805382 /loss/FRt1/detection_loss‚0.5748510056375163 /loss/FRt2/detection_loss‚0.4962487286166298 %loss/total_loss‚2.553598314331582 trainer/global_step‚5049 _step‚415œª€¾v¼ ð trainer/global_step‚5049 _step‚415 0loss/FRt2/assignment_loss‚0.7207613822968976 epoch‚12 % _timestamp‚1.7866600111961062e+09 _runtime‚ 1359.72602405 0loss/FRt1/assignment_loss‚0.7617371977805382 /loss/FRt1/detection_loss‚0.5748510056375163 /loss/FRt2/detection_loss‚0.4962487286166298 %loss/total_loss‚2.553598314331582ŸyùÝ’½ ‚(:‡ °‰ùÓÛÕÎâ gpu.0.gpu‚0.0 gpu.0.memory‚0, gpu.0.memoryAllocated‚7.0313262939453125) gpu.0.memoryAllocatedBytes‚ 6039863296 gpu.0.temp‚34.0 gpu.0.powerWatts‚119.48' gpu.0.enforcedPowerLimitWatts‚500.0 gpu.0.powerPercent‚23.896 gpu.0.smClock‚1410 gpu.0.memoryClock‚1593$ gpu.0.correctedMemoryErrors‚1955# gpu.0.uncorrectedMemoryErrors‚0Z“2bœ¾ ‚(:‘ °‰ùÓüÎöã) proc.memory.percent‚13.75126838684082 cpu‚26.969257707633194 network.sent‚ 6.785184e+06 network.recv‚ 2.77518e+06% disk./.usageGB‚486.97099685668945 disk.sda.out‚ 27.62890625 disk.sda1.out‚0 disk.sda2.out‚0" disk.nvme0n1.in‚17213.19140625& disk.nvme0n1.out‚52564.76318359375 disk.sda.in‚ 12917.8515625 disk.sda1.in‚0 disk.sda2.in‚0 disk.dm-1.in‚ 12917.8515625 disk.dm-1.out‚ 27.62890625 disk.drbd2165.in‚0 disk.drbd2165.out‚0# disk.dm-0.out‚52564.76318359375) disk./.usagePercent‚32.68103219493405( disk.nvme0c0n1.out‚52564.76318359375" proc.memory.rssMB‚ 2253.0078125 proc.cpu.threads‚17 disk.sda3.in‚ 12917.8515625 disk.sda3.out‚ 27.62890625$ disk.nvme0c0n1.in‚17213.19140625 disk.dm-0.in‚17213.19140625$ memory_percent‚65.45443534851074( proc.memory.availableMB‚ 5659.9453125- Âl2¿ - _wandbruntime‚1372  _runtime‚1372‡K­NÇÀ Á & lr-AdamW/pg1‚0.0012971377556896954 & lr-AdamW/pg2‚0.0012971377556896954 trainer/global_step‚5099 % _timestamp‚1.7866600240470128e+09 _runtime‚1372.577072984 _step‚416<ØÕµÌÁ Æ _runtime‚1372.577072984 _step‚416 & lr-AdamW/pg1‚0.0012971377556896954 & lr-AdamW/pg2‚0.0012971377556896954 trainer/global_step‚5099 % _timestamp‚1.7866600240470128e+09 ^æSo2 - _wandbruntime‚1372  _runtime‚1372@¾:½'à ·O WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17104896 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚138945749680128 UHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚70973440 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 5494538240 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 5368709120 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚18849793 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚18849791 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚262 IDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚140176558084608 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚17108898 PCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚58387380 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 5494538240 IBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚60 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚4287 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 1230850349568 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 1230808404480 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚322 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚140176617134592 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚139356732005432 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 4628521206 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 125829120 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚262 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚4280628 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 1301524686336 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚41945088 PIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚78 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚41945088 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚122448232656384 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 5368709120 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 4654602240 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚429 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚322 KDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚60 KBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚4876 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17104896 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚18849793 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚262 D;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚5264 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 1228283681848 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 831258624 PCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚59049984 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚18849791 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 4552982528 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚17104611 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚138128431284224 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚61478912 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 5494538240 _runtime‚1372.581896096 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚17108898 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 122306526 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚35958691 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 1228242333828 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚322 _step‚417 IBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚80 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚140176558084608 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚138945749680128 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 122905600 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 122905600 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 F>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚16014413 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚16014333 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚20643840 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚41348020 % _timestamp‚1.7866600240479712e+09 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚17104611 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚4287 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 F?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚60 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚138945766785024 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚209 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 845178880 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚121146778943488 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚138945766785024 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 4552982528 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚121146758299648 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚223 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚140176617134592 C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚4289 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 M@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚59049984 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 1230808404480 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 1301474356736 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚11733707 IDeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4289 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚138128448323584 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 4517085184 trainer/global_step‚5099 E8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚35954402 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 1230850349568 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚139356673618052 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 125829120 JBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚665 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚11733705 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚322 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 5368709120 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 125829120 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 4654602240 G>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4876 ¡0`Ä ¼O JBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚665 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚11733705 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚322 XIDeviceStatsMonitor.on_train_batch_start/reser;ßžš§&ved_bytes.large_pool.current‚ 5368709120 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 125829120 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 4654602240 G>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4876 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17104896 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚138945749680128 UHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚70973440 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 5494538240 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 5368709120 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚18849793 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚18849791 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚262 IDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚140176558084608 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚17108898 PCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚58387380 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 5494538240 IBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚60 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚4287 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 1230850349568 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 1230808404480 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚322 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚140176617134592 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚139356732005432 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 4628521206 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 125829120 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚262 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚4280628 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 1301524686336 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚41945088 PIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚78 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚41945088 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚122448232656384 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 5368709120 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 4654602240 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚429 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚322 KDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚60 KBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚4876 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17104896 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚18849793 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚262 D;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚5264 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 1228283681848 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 831258624 PCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚59049984 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚18849791 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 4552982528 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚17104611 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚138128431284224 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚61478912 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 5494538240 _runtime‚1372.581896096 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚17108898 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 122306526 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚35958691 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 1228242333828 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚322 _step‚417 IBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚80 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚140176558084608 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚138945749680128 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 122905600 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 122905600 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 F>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚16014413 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚16014333 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚20643840 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚41348020 % _timestamp‚1.7866600240479712e+09 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚17104611 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚4287 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 F?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚60 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚138945766785024 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚209 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 845178880 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚121146778943488 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚138945766785024 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 4552982528 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚121146758299648 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚223 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚140176617134592 C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚4289 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 M@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚59049984 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 1230808404480 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 1301474356736 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚11733707 IDeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4289 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚138128448323584 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 4517085184 trainer/global_step‚5099 E8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚35954402 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 1230850349568 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚139356673618052 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 125829120¡¥'9u2Å - _wandbruntime‚1372  _runtime‚1372ÇvÊOÆ&Æ ÀM GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 4517085184 _runtime‚1372.744957779 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654602240 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 1231039473152 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 4552982528 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 5368709120 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚138972072017920 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚429 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚35961242 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 5494538240 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚18853344 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚16017339 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚209 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚139383148371628 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚4295 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚41948160 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 1231081421312 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚138972089122816 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚35965537 B9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚5264 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚35961242 EDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚17112191 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚11735910 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚322 D=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚60 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚17112191 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚4281502 NADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚58387428 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 5494538240 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 5368709120 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 >5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5264 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚121169826316288 B6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚322 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17104896 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚139383089984200 PDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59053056 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚138972089122816 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 831258624 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 1301768665600 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚262 trainer/global_step‚5099 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚140203170544128 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 122905600 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚4293 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 125829120 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚46132224 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 4552982528 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 122905600 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚223 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚4293 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17104896 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 1231039473152 I@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚4876 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚122471594981888 IBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚60 D8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚262 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚16017414 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 1231081421312 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚121169805672448 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 1301722533376 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521206 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚41348068 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚322j Ë&Ç ÅM D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚11735910 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚322 D=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚60 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚17112191 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚4281502 NADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚58387428 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 5494538240 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 5368709120 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 >5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5264 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚121169826316288 B6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚322 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17104896 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚139383089984200 PDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59053056 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚138972089122816 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 831258624 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 1301768665600 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚262 trainer/global_step‚5099 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚140203170544128 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 122905600 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚4293 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 125829120 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚46132224 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 4552982528 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 122905600 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚223 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚4293 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17104896 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 1231039473152 I@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚4876 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚122471594981888 IBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚60 D8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚262 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚16017414 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 1231081421312 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚121169805672448 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 1301722533376 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521206 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚41348068 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚322 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 4517085184 _runtime‚1372.744957779 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654602240 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 1231039473152 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 4552982528 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 5368709120 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚138972072017920 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚429 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚35961242 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 5494538240 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚18853344 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚16017339 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚209 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚139383148371628 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚4295 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚41948160 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 1231081421312 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚138972089122816 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚35965537 B9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚5264 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚35961242 EDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚17112191¢'Óu 2È - _wandbruntime‚1372  _runtime‚1372²SsÉ í 0loss/FRt1/assignment_loss‚0.7796972283654575 /loss/FRt2/detection_loss‚0.4802729567290123 &loss/total_loss‚2.5680510754666637 epoch‚12 _runtime‚1372.745515753 0loss/FRt2/assignment_loss‚0.7656771070919667 /loss/FRt1/detection_loss‚0.5424037832802273 trainer/global_step‚5099 % _timestamp‚1.7866600242156856e+09 _step‚419{Yé§xÊ ò 0loss/FRt2/assignment_loss‚0.7656771070919667 /loss/FRt1/detection_loss‚0.5424037832802273 trainer/global_step‚5099 % _timestamp‚1.7866600242156856e+09 _step‚419 0loss/FRt1/assignment_loss‚0.7796972283654575 /loss/FRt2/detection_loss‚0.4802729567290123 &loss/total_loss‚2.5680510754666637 epoch‚12 _runtime‚1372.745515753£¨šô–Ë ‚(:‹ ¿‰ùÓ˜äá gpu.0.gpu‚63.0 gpu.0.memory‚12, gpu.0.memoryAllocated‚7.0313262939453125) gpu.0.memoryAllocatedBytes‚ 6039863296 gpu.0.temp‚34.0 gpu.0.powerWatts‚120.011' gpu.0.enforcedPowerLimitWatts‚500.0 gpu.0.powerPercent‚24.0022 gpu.0.smClock‚1410 gpu.0.memoryClock‚1593$ gpu.0.correctedMemoryErrors‚1955# gpu.0.uncorrectedMemoryErrors‚0¨œc‘Ì ‚(:† ¿‰ùÓ˜²ëâ network.sent‚ 6.855337e+06) disk./.usagePercent‚32.67797700102042 disk.sda.out‚ 27.69921875 disk.drbd2165.out‚0 disk.sda3.out‚ 27.69921875& proc.memory.availableMB‚ 5660.65625) proc.memory.percent‚13.75126838684082 network.recv‚ 2.80252e+06$ disk./.usageGB‚486.9254722595215 disk.sda.in‚ 12917.8515625 disk.sda2.out‚0 disk.dm-1.in‚ 12917.8515625 disk.dm-1.out‚ 27.69921875 disk.dm-0.in‚17213.19921875! disk.dm-0.out‚52571.322265625 disk.sda1.out‚0 disk.sda2.in‚0$ disk.nvme0n1.out‚52571.322265625 disk.sda3.in‚ 12917.8515625$ disk.nvme0c0n1.in‚17213.19921875& disk.nvme0c0n1.out‚52571.322265625# memory_percent‚65.4500961303711" proc.memory.rssMB‚ 2253.0078125 disk.sda1.in‚0" disk.nvme0n1.in‚17213.19921875 disk.drbd2165.in‚0 cpu‚26.97015545923565 proc.cpu.threads‚17jês`2Í - _wandbruntime‚1385  _runtime‚1385Ö«¢ûÇÎ Á trainer/global_step‚5149 % _timestamp‚1.7866600369468632e+09 _runtime‚1385.476925726 _step‚420 & lr-AdamW/pg1‚0.0012930091342946003 & lr-AdamW/pg2‚0.0012930091342946003% ÀÌÏ Æ & lr-AdamW/pg1‚0.0012930091342946003 & lr-AdamW/pg2‚0.0012930091342946003 trainer/global_step‚5149 % _timestamp‚1.7866600369468632e+09 _runtime‚1385.476925726 _step‚420¤•Â€†2Ð - _wandbruntime‚1385  _runtime‚1385 ¸™ùÑ ¶O E8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚36297154 @7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚5264 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 5368709120 WEDeviceStatsMonii‹ÚFÃ&tor.on_train_batch_start/active_bytes.small_pool.freed‚ 1242395170304 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 122905600 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚322 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 1242395170304 IDeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4289 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚4287 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚16165740 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚44038144 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 125829120 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 122905600 UHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚64681984 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 125829120 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 5494538240 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚262 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚141504320790016 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 1242437115392 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚41348020 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 F>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚19027443 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚322 KDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚60 M@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚59049984 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚61478912 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 1239845492456 D;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚5264 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 trainer/global_step‚5149 PCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚59049984 KBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚4876 _step‚421 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 4552982528 JBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚665 C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚4289 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚16165810 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚4321785 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 PCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚58387380 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚139437720961024 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚209 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚322 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚140677525105460 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 IBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚60 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚140261883674624 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚122299147583488 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚19027441 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚4321853 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 831258624 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 4654602240 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 845178880 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 4628521206 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚262 G>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4876 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚140677583492840 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚17274000 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚141504261740032 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17104896 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 4552982528 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚36301443 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 5368709120 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚17269713 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚322 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚41945088 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚140261866569728 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 4517085184 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 125829120 IDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 4654602240 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚141504320790016 IBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚70 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚122299126939648 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 1313767530496 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 5494538240 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚123612959152128 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚262 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚19027443 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚140261866569728 PIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚68 $ _timestamp‚1.786660036947846e+09 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚139437738000384 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 122306526 _runtime‚1385.481916012~“>Á'Ò »O _step‚421 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 4552982528 JBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚665 C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚4289 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚16165810 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚4321785 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 PCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚58387380 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚139437720961024 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚209 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚322 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚140677525105460 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 IBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚60 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚140261883674624 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚122299147583488 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚19027441 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚4321853 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 831258624 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 4654602240 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 845178880 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 4628521206 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚262 G>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4876 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚140677583492840 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚17274000 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚141504261740032 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17104896 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 4552982528 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚36301443 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 5368709120 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚17269713 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚322 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚41945088 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚140261866569728 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 4517085184 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 125829120 IDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 4654602240 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚141504320790016 IBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚70 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚122299126939648 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 1313767530496 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 5494538240 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚123612959152128 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚262 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚19027443 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚140261866569728 PIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚68 $ _timestamp‚1.786660036947846e+09 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚139437738000384 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 122306526 _runtime‚1385.481916012 E8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚36297154 @7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚5264 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 5368709120 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 1242395170304 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 122905600 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚322 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 1242395170304 IDeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4289 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚4287 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚16165740 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚44038144 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 125829120 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 122905600 UHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚64681984 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 125829120 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 5494538240 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚262 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚141504320790016 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 1242437115392 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚41348020 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 F>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚19027443 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚322 KDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚60 M@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚59049984 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚61478912 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 1239845492456 D;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚5264 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 trainer/global_step‚5149 PCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚59049984 KBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚4876¥áµÌ2Ó - _wandbruntime‚1385  _runtime‚13857ƒ—vÇ&Ô ÁM \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 1242668187136 H@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚665 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚4322731 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚44035072 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 EDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521206 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚262 >5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5264 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚41948160 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚141530874199552 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚20643840 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 1240076065116 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 1240034717048 K>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59053056 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚61478912 DDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚19030996 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 1314054827008 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 125829120 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚123636185104384 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17104896 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 4552982528 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚19030996 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚36308289 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚16168815 SFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚64678912 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚122322174312448 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚140703941471608 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17104896 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚4322655 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚141530815146496 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚17277293 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚140288188907520 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚140288206012416 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚16168893 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚322 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚17273000 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚41948160 E6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚322 trainer/global_step‚5149 P6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚322 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 5368709120 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚19030994 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654602240 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚209 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚262 _runtime‚1385.649460659 BDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654602240 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚209 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚262 _runtime‚1385.649460659 BDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 EDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521206 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚262 >5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5264 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚41948160 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚141530874199552 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚20643840 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 1240076065116 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 1240034717048 K>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59053056 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚61478912 DDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚19030996 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 1314054827008 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 125829120 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚123636185104384 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17104896 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 4552982528 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚19030996 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚36308289 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚16168815 SFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚64678912 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚122322174312448 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚140703941471608 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17104896 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚4322655 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚141530815146496 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚17277293 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚140288188907520 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚140288206012416 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚16168893 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚322 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚17273000 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚41948160 E6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚322 trainer/global_step‚5149 P6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚322 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0¦},²2Ö - _wandbruntime‚1385  _runtime‚1385aݵXq× ë _step‚423 0loss/FRt2/assignment_loss‚0.7941415401095879 /loss/FRt1/detection_loss‚0.5700452552964108 %loss/total_loss‚2.670338473880618 epoch‚12 _runtime‚1385.650517426 0loss/FRt1/assignment_loss‚0.7845601477668097 /loss/FRt2/detection_loss‚0.5215915307078095 trainer/global_step‚5149 $ _timestamp‚1.786660037120294e+09¯tY vØ ð %loss/total_loss‚2.670338473880618 epoch‚12 _runtime‚1385.650517426 0loss/FRt1/assignment_loss‚0.7845601477668097 /loss/FRt2/detection_loss‚0.5215915307078095 trainer/global_step‚5149 $ _timestamp‚1.786660037120294e+09 _step‚423 0loss/FRt2/assignment_loss‚0.7941415401095879 /loss/FRt1/detection_loss‚0.5700452552964108§øÆQ|–Ù ‚(:‹ ΉùÓÅÝá gpu.0.gpu‚64.0 gpu.0.memory‚12, gpu.0.memoryAllocated‚7.0313262939453125) gpu.0.memoryAllocatedBytes‚ 6039863296 gpu.0.temp‚35.0 gpu.0.powerWatts‚121.766' gpu.0.enforcedPowerLimitWatts‚500.0 gpu.0.powerPercent‚24.3532 gpu.0.smClock‚1410 gpu.0.memoryClock‚1593$ gpu.0.correctedMemoryErrors‚1955# gpu.0.uncorrectedMemoryErrors‚0(Ïå—Ú ‚(:Œ ΉùÓÚö²ã disk.sda.in‚ 12917.8515625 disk.sda1.out‚0 disk.nvme0n1.in‚ 17243.921875& disk.nvme0n1.out‚52687.86572265625 disk.drbd2165.out‚0 disk.dm-1.in‚ 12917.8515625 disk.dm-0.in‚ 17243.921875# disk.dm-0.out‚52687.86572265625" proc.memory.rssMB‚ 2253.0078125) proc.memory.percent‚13.75126838684082 proc.cpu.threads‚17 network.sent‚ 6.928998e+06 disk.sda3.out‚ 27.92578125" disk.nvme0c0n1.in‚ 17243.921875 disk.sda.out‚ 27.92578125 disk.dm-1.out‚ 27.92578125 disk.drbd2165.in‚0$ memory_percent‚65.45205116271973* disk./.usagePercent‚32.685373826707234 disk.sda3.in‚ 12917.8515625( disk.nvme0c0n1.out‚52687.86572265625 disk.sda1.in‚0 disk.sda2.out‚0( proc.memory.availableMB‚ 5660.3359375 cpu‚26.971318880749475 disk.sda2.in‚0 network.recv‚ 2.833561e+06$ disk./.usageGB‚487.03569030761720ß#2Û - _wandbruntime‚1398  _runtime‚1398‰’ÕÇÜ Á & lr-AdamW/pg2‚0.0012888456364375045 trainer/global_step‚5199 % _timestamp‚1.7866600504011652e+09 _runtime‚1398.931227346 _step‚424 & lr-AdamW/pg1‚0.0012888456364375045N]©ÌÝ Æ _step‚424 & lr-AdamW/pg1‚0.0012888456364375045 & lr-AdamW/pg2‚0.0012888456364375045 trainer/global_step‚5199 % _timestamp‚1.7866600504011652e+09 _runtime‚1398.931227346¨¬‰tü2Þ - _wandbruntime‚1398  _runtime‚1398Æcë¼'ß ¶O SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 845178880 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚140747027677184 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚4362934 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚124777604626432 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚123451516223488 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 125829120 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚262 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚142831965395456 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17104896 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4289 UHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚62584832 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚124777542041600 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 F>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚142831965395456 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 1254023881216 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 1251365955044 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚322 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚17439102 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚19205093 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚322 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 125829120 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 5368709120 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 1253981936128 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚41945088 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚140747010637824 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚262 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 5494538240 G>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4876 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚322 M@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚59049984 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚17434815 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚262 _runtime‚1398.932579991 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 4654602240 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚322 @7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚5264 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 1326088402944 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 1326046461952 F?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚60 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 4552982528 D;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚5264 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚16317141 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚11954205 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 5368709120 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚141998434980248 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 E8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚36639906 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚142832024445440 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 122905600 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚141578000564224 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 4552982528 JBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚665 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 831258624 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚19205091 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 122306526 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 125829120 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 122905600 BDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 PCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚58387380 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚41348020 PIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚64 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚142832024445440 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 5494538240 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚19205093 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚4287 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17104896 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚209 KDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚60 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚36644195 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚06ÌRÁ'à »O F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚124777604626432 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚123451516223488 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 125829120 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚262 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚142831965395456 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17104896 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4289 UHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚62584832 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚124777542041600 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 F>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚142831965395456 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 1254023881216 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 1251365955044 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚322 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚17439102 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚19205093 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚322 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 125829120 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 5368709120 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 1253981936128 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚41945088 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚140747010637824 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚262 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 5494538240 G>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4876 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚322 M@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚59049984 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚17434815 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚262 _runtime‚1398.932579991 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 4654602240 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚322 @7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚5264 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 1326088402944 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 1326046461952 F?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚60 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 4552982528 D;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚5264 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚16317141 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚11954205 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 5368709120 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚141998434980248 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 E8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚36639906 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚142832024445440 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 122905600 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚141578000564224 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 4552982528 JBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚665 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 831258624 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚19205091 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 122306526 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 125829120 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 122905600 BDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 PCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚58387380 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚41348020 PIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚64 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚142832024445440 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 5494538240 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚19205093 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚4287 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17104896 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚209 KDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚60 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚36644195 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 845178880 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚140747027677184 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚4362934 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0©_L('2á - _wandbruntime‚1399  _runtime‚1399‡#µ²F â ÁM XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 1251596527656 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 5368709120 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚262 _runtime‚1399.102561735 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚124800897354752 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 1254254952960 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 1254213004800 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 1254254952960 I@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚4876 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚36646746 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 122905600 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚429 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚322 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 5368709120 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚322 G@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚60 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚17442395 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚11956410 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚142858577854976 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 1254213004800 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 122905600 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 5494538240 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚4293 G:DeviceStatsMonitor.on_train_batch_end/active.arq'ell.allocated‚36651041 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚17438102 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚41348068 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 125829120 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 P6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚322 IBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚60 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 4552982528 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚4363716 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚124800830578688 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 831258624 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 1326333758464 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚262 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚19208644 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17104896 EDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚11956412 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚46132224 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 5368709120 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚36646746 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚209 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 1326287626240 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 5494538240 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚322 D=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚60 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚141604322902016 SFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚66776064 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚20643840 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 122306526 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654602240 EDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521206 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17104896 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚223 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚123474542952448 B5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5264 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚16320057 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 trainer/global_step‚5199 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚141604305797120 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚141604305797120 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚140773213470720 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 1251637875724 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 125829120 K>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59053056 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚36651041 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚4363647 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚4293 B9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚5264 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 4517085184 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚16320128 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚142024792959016çéŠÌ&ã ÆM ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚20643840 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 122306526 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654602240 EDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521206 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17104896 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚223 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚123474542952448 B5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5264 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚16320057 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 trainer/global_step‚5199 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚141604305797120 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚141604305797120 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚140773213470720 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 1251637875724 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 125829120 K>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59053056 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚36651041 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚4363647 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚4293 B9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚5264 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 4517085184 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚16320128 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚142024792959016 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 1251596527656 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 5368709120 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚262 _runtime‚1399.102561735 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚124800897354752 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 1254254952960 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 1254213004800 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 1254254952960 I@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚4876 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚36646746 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 122905600 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚429 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚322 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 5368709120 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚322 G@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚60 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚17442395 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚11956410 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚142858577854976 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 1254213004800 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 122905600 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 5494538240 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚4293 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚36651041 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚17438102 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚41348068 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 125829120 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 P6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚322 IBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚60 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 4552982528 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚4363716 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚124800830578688 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 831258624 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 1326333758464 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚262 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚19208644 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17104896 EDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚11956412 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚46132224 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 5368709120 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚36646746 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚209 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 1326287626240 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 5494538240 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚322 D=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚60 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚141604322902016 SFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚66776064ªÃÕƒø2ä - _wandbruntime‚1399  _runtime‚1399¬3þñqå ë 0loss/FRt2/assignment_loss‚0.7719993077181827 /loss/FRt2/detection_loss‚0.5288071078290356 epoch‚12 _step‚427 0loss/FRt1/assignment_loss‚0.7286759839246407 /loss/FRt1/detection_loss‚0.5190394193538392 %loss/total_loss‚2.548521818825698 trainer/global_step‚5199 $ _timestamp‚1.786660050573234e+09 _runtime‚1399.103472395y­]væ ð /loss/FRt1/detection_loss‚0.5190394193538392 %loss/total_loss‚2.548521818825698 trainer/global_step‚5199 $ _timestamp‚1.786660050573234e+09 _runtime‚1399.103472395 0loss/FRt2/assignment_loss‚0.7719993077181827 /loss/FRt2/detection_loss‚0.5288071078290356 epoch‚12 _step‚427 0loss/FRt1/assignment_loss‚0.7286759839246407«ÃÁ†q2ç - _wandbruntime‚1404  _runtime‚1404P°FÚè Ô 'particle/accuracy‚0.653506338596344 epoch‚12 3Purity/1FRt/event_proportion‚0.6297432780265808 +particle/specificity‚0.6736034154891968 2Purity/*FRt/event_proportion‚0.669805645942688 -Purity/1FRt/FRt_purity‚0.4590460956096649 +jet/accuracy_2_of_2‚0.20074376463890076 *particle/sensitivity‚0.616942286491394 trainer/global_step‚5199 *loss/val_total_loss‚2.5191619783598393 +validation_accuracy‚0.20074376463890076 6validation_average_jet_accuracy‚0.3123374283313751 .particle/accuracy_2_of_2‚0.620563805103302 'particle/f_score‚0.5582449436187744 Purity/0FRt/event_purity‚1 3Purity/0FRt/event_proportion‚0.3301943838596344 2Purity/2FRt/event_proportion‚0.040062315762043 *jet/accuracy_1_of_2‚0.5576484203338623 particle/accuracy_1_of_1‚1 _runtime‚1404.446305749 0Purity/*FRt/event_purity‚0.45541587471961975 *jet/accuracy_1_of_1‚0.3080132007598877 0Purity/2FRt/event_purity‚0.39627355337142944 /particle/accuracy_1_of_2‚0.8917048573493958 $ _timestamp‚1.786660055915711e+09 _step‚428 -Purity/*FRt/FRt_purity‚0.4682067632675171 /Purity/1FRt/event_purity‚0.4590460956096649 -Purity/2FRt/FRt_purity‚0.5385497808456421Z^"õßé Ù 2Purity/*FRt/event_proportion‚0.669805645942688 -Purity/1FRt/FRt_purity‚0.4590460956096649 +jet/accuracy_2_of_2‚0.20074376463890076 *particle/sensitivity‚0.616942286491394 trainer/global_step‚5199 *loss/val_total_loss‚2.5191619783598393 +validation_accuracy‚0.20074376463890076 6validation_average_jet_accuracy‚0.3123374283313751 .particle/accuracy_2_of_2‚0.620563805103302 'particle/f_score‚0.5582449436187744 Purity/0FRt/event_purity‚1 3Purity/0FRt/event_proportion‚0.3301943838596344 2Purity/2FRt/event_proportion‚0.040062315762043 *jet/accuracy_1_of_2‚0.5576484203338623 particle/accuracy_1_of_1‚1 _runtime‚1404.446305749 0Purity/*FRt/event_purity‚0.45541587471961975 *jet/accuracy_1_of_1‚0.3080132007598877 0Purity/2FRt/event_purity‚0.39627355337142944 /particle/accuracy_1_of_2‚0.8917048573493958 $ _timestamp‚1.786660055915711e+09 _step‚428 -Purity/*FRt/FRt_purity‚0.4682067632675171 /Purity/1FRt/event_purity‚0.4590460956096649 -Purity/2FRt/FRt_purity‚0.5385497808456421 'particle/accuracy‚0.653506338596344 epoch‚12 3Purity/1FRt/event_proportion‚0.6297432780265808 +particle/specificity‚0.6736034154891968¬n#Å¡ê ‚(:– ݉ùÓôå‚á gpu.0.gpu‚55.0 gpu.0.memory‚11, gpu.0.memoryAllocated‚7.0313262939453125) gpu.0.memoryAllocatedBytes‚ 6039863296 gpu.0.temp‚35.0 gpu.0.powerWatts‚209.241' gpu.0.enforcedPowerLimitWatts‚500.0) gpu.0.powerPercent‚41.848200000000006 gpu.0.smClock‚1410 gpu.0.memoryClock‚1593$ gpu.0.correctedMemoryErrors‚1955# gpu.0.uncorrectedMemoryErrors‚0¦rë ‚(:Š ݉ùÓØ’Çä disk.nvme0n1.in‚ 17243.984375 disk.dm-1.out‚ 28.14453125 disk.drbd2165.in‚0 disk.sda3.in‚ 12917.8515625( proc.memory.availableMB‚ 5558.9609375# proc.memory.rssMB‚ 2354.21484375 disk.sda1.out‚0 disk.sda2.out‚0 disk.dm-1.in‚ 12917.8515625 disk.dm-0.in‚ 17243.984375" disk.dm-0.out‚52723.8212890625" disk.nvme0c0n1.in‚ 17243.984375' disk.nvme0c0n1.out‚52723.8212890625# memory_percent‚66.0707950592041 network.sent‚ 7.004076e+06 network.recv‚ 2.866378e+06 disk.sda.in‚ 12917.8515625% disk.nvme0n1.out‚52723.8212890625 disk.drbd2165.out‚0 cpu‚26.989884527361657 proc.cpu.threads‚17% disk./.usageGB‚487.03936767578125 disk.sda.out‚ 28.14453125 disk.sda2.in‚0 disk.sda3.out‚ 28.14453125* proc.memory.percent‚14.368987083435059) disk./.usagePercent‚32.68562061797024 disk.sda1.in‚0u™*2ì - _wandbruntime‚1417  _runtime‚14177®¼Çí Á % _timestamp‚1.7866600694017003e+09 _runtime‚1417.931893352 _step‚429 & lr-AdamW/pg1‚0.0012846475295320958 & lr-AdamW/pg2‚0.0012846475295320958 trainer/global_step‚5249b(Ìî Æ _step‚429 & lr-AdamW/pg1‚0.0012846475295320958 & lr-AdamW/pg2‚0.0012846475295320958 trainer/global_step‚5249 % _timestamp‚1.7866600694017003e+09 _runtime‚1417.931893352­r`2ï - _wandbruntime‚1417  _runtime‚1417ÙGï±'ð ·O PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚19443347 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚209 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 1266993737758 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 5494538240 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚262 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚144554673992192 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 4517085184 F>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚20643840 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 122306526 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 M@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚59049984 IDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 5368709120 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 1269678373376 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚322 F?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚60 C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚4289 G>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4876 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17104896 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 122905600 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚37069745 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 IBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚60 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 IBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚68 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚429 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 1267035085778 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 1269636428288 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚223 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚126245598495744 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚124903155171328 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚124903134527488 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 trainer/global_step‚5249 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 4654602240 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚41945088 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 1342443324416 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 PCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚59049984 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 831258624 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚143704251931602 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚41348020 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 4654602240 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚144554733042176 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚12105069 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚4287 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚143285037563904 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 1269678373376 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚19443349 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚143285037563904 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 E8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚37065456 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4289 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚17622109 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚126245527522304 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚322 % _timestamp‚1.7866600694027927e+09 D;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚5264 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚16516828 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚50329600 _step‚430 IDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚144554673992192 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 4517085184 F>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚20643840 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 122306526 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 M@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚59049984 IDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 5368709120 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 1269678373376 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚322 F?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚60 C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚4289 G>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4876 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17104896 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 122905600 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚37069745 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 IBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚60 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 IBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚68 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚429 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 1267035085778 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 1269636428288 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚223 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚126245598495744 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚124903155171328 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚124903134527488 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 trainer/global_step‚5249 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 4654602240 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚41945088 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 1342443324416 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 PCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚59049984 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 831258624 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚143704251931602 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚41348020 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 4654602240 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚144554733042176 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚12105069 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚4287 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚143285037563904 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 1269678373376 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚19443349 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚143285037563904 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 E8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚37065456 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4289 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚17622109 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚126245527522304 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚322 % _timestamp‚1.7866600694027927e+09 D;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚5264 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚16516828 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚50329600 _step‚430®=GÓj2ò - _wandbruntime‚1418  _runtime‚1418L^Ç&ó ÁM E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚262 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚4293 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚41948160 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚144581286451712 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚4293 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚209 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚126268889847808 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 _runtime‚1418.101407464 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 5368709120 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 122905600 NGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚72 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 122306526 _step‚431 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚17629689 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 SFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚66776064 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚124926181900288 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚262 IBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚60 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚429 NADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚59053056 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚46132224 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚37076591 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚37072296 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚142463385600000 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚12107274 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 1269867496960 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654602240 >5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5264 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚19446900 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚19446902 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚17625396 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚143730668297798 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 5494538240 D=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚60 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 5368709120 trainer/global_step‚5249 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 125829120 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 1269867496960 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚37076591 G@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚74 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚126268823071744 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 125829120 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚4295 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚61478912 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 5494538240 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 5368709120 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚322 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚142463402639360 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 1269909445120 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚16519810 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚12107276 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚37072296 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 122905600 EDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521206 % _timestamp‚1.7866600695710547e+09 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚124926202544128 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 4552982528 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚41948160 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚223 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 831258624 BDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59053056 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚19446902 H@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚665 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 125829120 D6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚322 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 NADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚58387428 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚143311359901696 I@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚4876 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚20643840 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚143730609910370 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚143311377006592 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚16519884 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚322 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚41348068 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 EDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 1342687303680 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚143311359901696 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 4552982528 B9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚5264 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 4517085184 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚144581227398656-@ÄBÌ&ô ÆM M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654602240 >5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5264 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚19446900 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚19446902 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚17625396 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚143730668297798 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 5494538240 D=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚60 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 5368709120 trainer/global_step‚5249 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 125829120 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 1269867496960 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚37076591 G@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚74 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚126268823071744 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 125829120 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚4295 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚61478912 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 5494538240 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 5368709120 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚322 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚142463402639360 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 1269909445120 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚16519810 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚12107276 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚37072296 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 122905600 EDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521206 % _timestamp‚1.7866600695710547e+09 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚124926202544128 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 4552982528 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚41948160 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚223 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 831258624 BDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59053056 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚19446902 H@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚665 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 125829120 D6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚322 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 NADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚58387428 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚143311359901696 I@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚4876 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚20643840 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚143730609910370 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚143311377006592 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚16519884 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚322 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚41348068 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 EDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 1342687303680 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚143311359901696 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 4552982528 B9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚5264 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 4517085184 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚144581227398656 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚262 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚4293 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚41948160 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚144581286451712 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚4293 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚209 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚126268889847808 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 _runtime‚1418.101407464 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 5368709120 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 122905600 NGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚72 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 122306526 _step‚431 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚17629689 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 SFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚66776064 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚124926181900288 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚262 IBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚60 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚429 NADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚59053056 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚46132224 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚37076591 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚37072296 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚142463385600000 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚12107274 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 1269867496960¯Ɔ¹p2õ - _wandbruntime‚1418  _runtime‚1418i´±Ésö í 0loss/FRt1/assignment_loss‚0.7230047617372701 /loss/FRt1/detection_loss‚0.5333703901583515 /loss/FRt2/detection_loss‚0.5039026436140152 epoch‚13 % _timestamp‚1.7866600695721707e+09 _runtime‚1418.102076457 0loss/FRt2/assignment_loss‚0.7318805543532998 &loss/total_loss‚2.4921583498629367 trainer/global_step‚5249 _step‚432â±¹x÷ ò /loss/FRt2/detection_loss‚0.5039026436140152 epoch‚13 % _timestamp‚1.7866600695721707e+09 _runtime‚1418.102076457 0loss/FRt2/assignment_loss‚0.7318805543532998 &loss/total_loss‚2.4921583498629367 trainer/global_step‚5249 _step‚432 0loss/FRt1/assignment_loss‚0.7230047617372701 /loss/FRt1/detection_loss‚0.5333703901583515°t¦M»–ø ‚(:‹ ì‰ùÓÉ›á gpu.0.gpu‚57.0 gpu.0.memory‚10, gpu.0.memoryAllocated‚7.0313262939453125) gpu.0.memoryAllocatedBytes‚ 6039863296 gpu.0.temp‚34.0 gpu.0.powerWatts‚120.839' gpu.0.enforcedPowerLimitWatts‚500.0 gpu.0.powerPercent‚24.1678 gpu.0.smClock‚1410 gpu.0.memoryClock‚1593$ gpu.0.correctedMemoryErrors‚1955# gpu.0.uncorrectedMemoryErrors‚0ù÷½˜ù ‚(: ì‰ùÓ©ß«ä disk.sda1.in‚0 disk.sda2.out‚0 network.sent‚ 7.076093e+06! disk.nvme0n1.in‚ 17252.8671875 disk.dm-1.out‚ 28.484375 disk.drbd2165.out‚0 disk.sda1.out‚0$ memory_percent‚66.07084274291992' proc.memory.availableMB‚ 5558.953125 proc.cpu.threads‚17 network.recv‚ 2.894191e+06 disk.sda3.out‚ 28.484375* proc.memory.percent‚14.368987083435059 cpu‚26.990337956646133) disk./.usagePercent‚32.68553715951408& disk.nvme0n1.out‚52753.71826171875 disk.drbd2165.in‚0# disk.dm-0.out‚52753.71826171875# disk.nvme0c0n1.in‚ 17252.8671875 disk.sda.out‚ 28.484375 disk.sda2.in‚0# proc.memory.rssMB‚ 2354.21484375% disk./.usageGB‚487.03812408447266 disk.dm-1.in‚12938.71484375 disk.dm-0.in‚ 17252.8671875 disk.sda3.in‚12938.71484375( disk.nvme0c0n1.out‚52753.71826171875 disk.sda.in‚12938.71484375\ÆŽ2ú - _wandbruntime‚1430  _runtime‚1430’²ãÇû Á _runtime‚1430.700423565 _step‚433 & lr-AdamW/pg1‚0.0012804150832149356 & lr-AdamW/pg2‚0.0012804150832149356 trainer/global_step‚5299 % _timestamp‚1.7866600821701605e+09#+¦6ü Æ _runtime‚1430.700423565 _step‚433 & DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4876 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 4654602240 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 125829120 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚322 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 4552982528 PCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚59049984 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 122905600 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 1281265139200 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚16668196 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 1278596896386 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 122306526 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚4287 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚126055523811328 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 831258624 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚145025103419010 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚41348020 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚19620997 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚37412497 JBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚665 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚262 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 1281223194112 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 KDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚60 _step‚434 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 125829120 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚223 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚4452877 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 845178880 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 5494538240 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 4654602240 IDeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4289 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚144601154453504 UHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚68876288 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚322 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚262 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚17791498 PIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚72 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 125829120 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17104896 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚41945088 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚4452949 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 5368709120 IBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚60 F>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 IDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚145882377647616 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚41945088 IBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚74 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚144601171558400 D;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚5264 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚429 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚145025045031630iÜJÉÁ'ÿ »O TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 125829120 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17104896 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚41945088 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚4452949 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 5368709120 IBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚60 F>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 IDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚145882377647616 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚41945088 IBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚74 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚144601171558400 D;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚5264 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚429 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚145025045031630 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 4628521206 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 G>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4876 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 4654602240 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 125829120 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚322 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 4552982528 PCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚59049984 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 122905600 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 1281265139200 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚16668196 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 1278596896386 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 122306526 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚4287 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚126055523811328 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 831258624 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚145025103419010 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚41348020 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚19620997 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚37412497 JBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚665 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚262 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 1281223194112 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 KDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚60 _step‚434 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 125829120 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚223 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚4452877 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 845178880 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 5494538240 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 4654602240 IDeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4289 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚144601154453504 UHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚68876288 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚322 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚262 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚17791498 PIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚72 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0²°ïÑ2€- _wandbruntime‚1430  _runtime‚1430õµ”Ç&ÁM PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 122905600 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚143772692316160 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 1278827469046 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 5368709120 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚41948160 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚127433539411968 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚4295 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 845178880 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚17790498 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚37415048 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚143772675276800 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 5494538240 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 5368709120 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚145908931054080 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚209 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 831258624 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 _runtime‚1430.870031045 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 B9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚5264 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚12217526 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 5494538240 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 EDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚16671214 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚322 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚145051519785206 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 5494538240 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚322 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚262 DDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521206 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚145908990107136 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17104896 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 IBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚60 NADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚58387428 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚262 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚4293 PDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59053056 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 1281454262784 SFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚68873216 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 1354968227840 trainer/global_step‚5299 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚144627493896192 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 122306526 >5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5264 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 125829120 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 D=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚60 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚19624550 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 4654602240 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚16671290 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚429 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚127433470538752 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚17794791 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17104896 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚144627476791296 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚223 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚322 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚322 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 1281496210944 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚19624552 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚20643840 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚126078550540288 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚262 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 1281496210944 BDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚37419343 NGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚74 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚145051461397778 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654602240 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚17794791 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚126078571184128 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚4293 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚37419343 I@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚4876 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 1354919998464 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚41348068 % _timestamp‚1.7866600823393052e+09 _step‚435 EDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚37419343 NGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚74 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚145051461397778 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654602240 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚17794791 cODevicÿ‘‘;eStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚126078571184128 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚4293 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚37419343 I@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚4876 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 1354919998464 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚41348068 % _timestamp‚1.7866600823393052e+09 _step‚435 EDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 EDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚16671214 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚322 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚145051519785206 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 5494538240 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚322 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚262 DDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521206 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚145908990107136 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17104896 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 IBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚60 NADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚58387428 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚262 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚4293 PDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59053056 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 1281454262784 SFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚68873216 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 1354968227840 trainer/global_step‚5299 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚144627493896192 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 122306526 >5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5264 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 125829120 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 D=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚60 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚19624550 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 4654602240 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚16671290 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚429 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚127433470538752 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚17794791 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17104896³Äyv›2ƒ- _wandbruntime‚1430  _runtime‚1430¤Q7r„ì .loss/FRt1/detection_loss‚0.563028932051232 &loss/total_loss‚2.8718232220889157 epoch‚13 % _timestamp‚1.7866600823407762e+09 _runtime‚1430.870624125 _step‚436 0loss/FRt1/assignment_loss‚0.9624558411899016 0loss/FRt2/assignment_loss‚0.8304677341170711 /loss/FRt2/detection_loss‚0.5158707147307113 trainer/global_step‚5299¶‹³¨w…ñ % _timestamp‚1.7866600823407762e+09 _runtime‚1430.870624125 _step‚436 0loss/FRt1/assignment_loss‚0.9624558411899016 0loss/FRt2/assignment_loss‚0.8304677341170711 /loss/FRt2/detection_loss‚0.5158707147307113 trainer/global_step‚5299 .loss/FRt1/detection_loss‚0.563028932051232 &loss/total_loss‚2.8718232220889157 epoch‚13´ó?p–†‚(:‹ û‰ùÓ”¢üà gpu.0.gpu‚56.0 gpu.0.memory‚11, gpu.0.memoryAllocated‚7.0313262939453125) gpu.0.memoryAllocatedBytes‚ 6039863296 gpu.0.temp‚34.0 gpu.0.powerWatts‚120.531' gpu.0.enforcedPowerLimitWatts‚500.0 gpu.0.powerPercent‚24.1062 gpu.0.smClock‚1410 gpu.0.memoryClock‚1593$ gpu.0.correctedMemoryErrors‚1955# gpu.0.uncorrectedMemoryErrors‚0(n²œ‡‚(:‘ û‰ùÓœÏã network.recv‚ 2.912627e+06$ disk./.usageGB‚487.0638999938965 disk.dm-1.in‚ 13997.0703125 disk.dm-0.in‚17269.41796875 disk.sda3.out‚ 28.6953125 proc.cpu.threads‚17# disk.dm-0.out‚52773.50732421875 disk.sda3.in‚ 13997.0703125 disk.sda.in‚ 13997.0703125* proc.memory.percent‚14.368987083435059 cpu‚26.991666863736835* disk./.usagePercent‚32.687267002422935 disk.dm-1.out‚ 28.6953125 disk.drbd2165.out‚0$ disk.nvme0c0n1.in‚17269.41796875 disk.sda.out‚ 28.6953125 disk.sda1.in‚0$ memory_percent‚66.07136726379395( proc.memory.availableMB‚ 5558.8671875 disk.drbd2165.in‚0( disk.nvme0c0n1.out‚52773.50732421875 disk.sda1.out‚0 disk.sda2.in‚0 disk.sda2.out‚0" disk.nvme0n1.in‚17269.41796875& disk.nvme0n1.out‚52773.50732421875# proc.memory.rssMB‚ 2354.21484375 network.sent‚ 7.137787e+06Ao`2ˆ- _wandbruntime‚1443  _runtime‚1443=)ƉÀ & lr-AdamW/pg1‚0.0012761485693281419 & lr-AdamW/pg2‚0.0012761485693281419 trainer/global_step‚5349 $ _timestamp‚1.786660094591899e+09 _runtime‚1443.123963505 _step‚437%¯ ËŠÅ & lr-AdamW/pg1‚0.0012761485693281419 & lr-AdamW/pg2‚0.0012761485693281419 trainer/global_step‚5349 $ _timestamp‚1.786660094591899e+09 _runtime‚1443.123963505 _step‚437µ5—*2‹- _wandbruntime‚1443  _runtime‚1443 ëkz½'Œ·O VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚147210140353024 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚17956600 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 1366990277632 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 122306526 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17104896 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚16819510 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚4494014 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 845178880 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 4552982528 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚41348020 trainer/global_step‚5349 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚12325569 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17104896 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚19798647 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 4654602240 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚4493941 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚127207892451328 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 4517085184 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 5494538240 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 5494538240 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚322 E8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚37750960 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚145917271343104 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 4628521206 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 1292851905024 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 122905600 PCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚59049984 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4289 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚4287 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 125829120 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚41945088 IDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚147210081303040 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 122905600 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚37755249 PCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚58387380 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 1290158706994 G>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4876 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚322 IBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚60 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚17956600 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 4654602240 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 1292851905024 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚20643840 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚4289 @7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚5264 M@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚59049984 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚147210140353024 IBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚75 _runtime‚1443.124071319 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚145917288448000 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚146345896519038 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚322 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚145917271343104 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 831258624 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 1366937850880 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚61478912 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚262 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 5368709120 % _timestamp‚1.7866600945925891e+09 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 4552982528 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚41945088 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚17952313 KBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚4876 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚262 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 5368709120 IDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚147210081303040 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 1290117358974 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 5494538240 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 5368709120 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚322 F?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚60 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚19798649 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 1292809959936 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚262 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0Wà-ÂÂ'¼O G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚429 KDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚60 _step‚438 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚17952313 JBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚665 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚223 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 D;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚5264 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚19798647 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚128574809658368 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚127207871807488 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚4287 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚128574882728960 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 F>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚147210081303040 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 1290117358974 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 5494538240 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 5368709120 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚322 F?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚60 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚19798649 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 1292809959936 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚262 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚147210140353024 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚17956600 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 1366990277632 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 122306526 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17104896 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚16819510 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚4494014 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 845178880 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 4552982528 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚41348020 trainer/global_step‚5349 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚12325569 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17104896 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚19798647 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 4654602240 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚4493941 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚127207892451328 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 4517085184 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 5494538240 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 5494538240 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚322 E8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚37750960 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚145917271343104 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 4628521206 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 1292851905024 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 122905600 PCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚59049984 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4289 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚4287 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 125829120 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚41945088 IDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚147210081303040 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 122905600 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚37755249 PCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚58387380 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 1290158706994 G>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4876 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚322 IBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚60 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚17956600 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 4654602240 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 1292851905024 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚20643840 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚4289 @7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚5264 M@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚59049984 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚147210140353024 IBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚75 _runtime‚1443.124071319 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚145917288448000 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚146345896519038 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚322 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚145917271343104 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 831258624 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 1366937850880 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚61478912 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚262 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 5368709120 % _timestamp‚1.7866600945925891e+09 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 4552982528 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚41945088 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚17952313 KBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚4876 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚262 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 5368709120 IDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 >5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5264 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚145943610785792 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚37757800 B6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚322 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 5494538240 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚322 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 4552982528 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 125829120 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚37757800 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚4494776 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚41348068 \JDeviceStatsMonitor.on_train_batch_end/allocated_byt \N‰Ves.small_pool.allocated‚ 1293082976768 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚20643840 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚61478912 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 5368709120 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚46132224 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚429 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 4552982528 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚17955600 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚128598106256384 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 125829120 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚37762095 I@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚4876 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 1293082976768 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 122905600 NADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚59053056 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚146372312885186 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 _step‚439 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚209 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 4517085184 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 5494538240 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 122905600 G@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚66 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚12327774 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521206 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚145943593680896 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚41948160 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 831258624 D=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚60 _runtime‚ 1443.28446202 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚4293 SFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚66776064 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚127230919180288 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 5494538240 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 122306526 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚17955600 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 1367187076096 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚19802200 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚17959893 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 4654602240 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 1293041028608 G@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚60 K>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59053056 NGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚64 NADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚58387428 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚145943610785792 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 1293041028608 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚19802202 H@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚665 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚147236634709504 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚145943593680896 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚4494840 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚127230939824128 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 trainer/global_step‚5349 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚146372371272614 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚262 DDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚16822550 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚223 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 1367233208320 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚322 % _timestamp‚1.7866600947541583e+09 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚37762095 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚17959893 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚12327776 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚4295 EDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654602240 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚128598173032448 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 5368709120 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 IBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚60 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚145081964953600 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚147236693762560 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚322 B9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚5264 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚19802200 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 845178880 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 125829120 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17104896 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 1290389279654 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 1290347931586 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 5368709120 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚147236693762560ÑÄÁË&ÅM H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚262 DDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚16822550 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚223 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 1367233208320 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚322 % _timestamp‚1.7866600947541583e+09 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚37762095 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚17959893 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚12327776 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚4295 EDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654602240 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚128598173032448 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 5368709120 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 IBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚60 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚145081964953600 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚147236693762560 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚322 B9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚5264 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚19802200 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 845178880 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 125829120 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17104896 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 1290389279654 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 1290347931586 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 5368709120 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚147236693762560 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 PDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 >5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5264 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚145943610785792 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚37757800 B6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚322 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 5494538240 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚322 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 4552982528 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 125829120 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚37757800 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚4494776 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚41348068 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 1293082976768 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚20643840 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚61478912 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 5368709120 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚46132224 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚429 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 4552982528 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚17955600 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚128598106256384 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 125829120 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚37762095 I@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚4876 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 1293082976768 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 122905600 NADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚59053056 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚146372312885186 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 _step‚439 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚209 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 4517085184 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 5494538240 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 122905600 G@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚66 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚12327774 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521206 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚145943593680896 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚41948160 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 831258624 D=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚60 _runtime‚ 1443.28446202 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚4293 SFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚66776064 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚127230919180288 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 5494538240 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 122306526 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚17955600 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 1367187076096 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚19802200 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚17959893 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 4654602240 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 1293041028608 G@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚60 K>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59053056 NGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚64 NADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚58387428 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚145943610785792 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 1293041028608 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚19802202 H@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚665 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚147236634709504 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚145943593680896 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚4494840 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚127230939824128 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 trainer/global_step‚5349 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚146372371272614·1ñÖ2‘- _wandbruntime‚1443  _runtime‚1443ÑTÙr’ì 0loss/FRt1/assignment_loss‚0.8099815511564386 0loss/FRt2/assignment_loss‚0.7892103340784159 /loss/FRt2/detection_loss‚0.5307781996022498 trainer/global_step‚5349 .loss/FRt1/detection_loss‚0.559212496056967 &loss/total_loss‚2.6891825808940713 epoch‚13 % _timestamp‚1.7866600947551908e+09 _runtime‚1443.285312236 _step‚440Žx¶w“ñ .loss/FRt1/detection_loss‚0.559212496056967 &loss/total_loss‚2.6891825808940713 epoch‚13 % _timestamp‚1.7866600947551908e+09 _runtime‚1443.285312236 _step‚440 0loss/FRt1/assignment_loss‚0.8099815511564386 0loss/FRt2/assignment_loss‚0.7892103340784159 /loss/FRt2/detection_loss‚0.5307781996022498 trainer/global_step‚5349¸™€*¬–”‚(:‹ ŠŠùÓ¸éŽá gpu.0.gpu‚63.0 gpu.0.memory‚12, gpu.0.memoryAllocated‚7.0313262939453125) gpu.0.memoryAllocatedBytes‚ 6039863296 gpu.0.temp‚36.0 gpu.0.powerWatts‚231.892' gpu.0.enforcedPowerLimitWatts‚500.0 gpu.0.powerPercent‚46.3784 gpu.0.smClock‚1410 gpu.0.memoryClock‚1593$ gpu.0.correctedMemoryErrors‚1955# gpu.0.uncorrectedMemoryErrors‚0P ÿ”•‚(:‰ ŠŠùÓǸÔã disk.dm-1.in‚14134.67578125 disk.sda.in‚14134.67578125 disk.sda1.out‚0$ memory_percent‚66.07439517974854# proc.memory.rssMB‚ 2354.21484375) disk./.usagePercent‚32.68991847245506 disk.sda2.out‚0 disk.dm-1.out‚ 29.08984375 disk.nvme0c0n1.in‚ 17292.8125) proc.memory.availableMB‚ 5558.37109375* proc.memory.percent‚14.368987083435059 network.sent‚ 7.204701e+06 network.recv‚ 2.935876e+06 disk.sda2.in‚0 disk.drbd2165.out‚0 disk.dm-0.in‚ 17292.8125 disk.sda3.out‚ 29.08984375' disk.nvme0c0n1.out‚52814.6064453125 disk.sda.out‚ 29.08984375% disk./.usageGB‚487.10340881347656% disk.nvme0n1.out‚52814.6064453125 disk.drbd2165.in‚0" disk.dm-0.out‚52814.6064453125 disk.sda3.in‚14134.67578125 disk.sda1.in‚0 cpu‚26.993238872798074 proc.cpu.threads‚17 disk.nvme0n1.in‚ 17292.8125õºå'2–- _wandbruntime‚1455  _runtime‚14559½.Å—¿ % lr-AdamW/pg1‚0.001271848261901929 % lr-AdamW/pg2‚0.001271848261901929 trainer/global_step‚5399 % _timestamp‚1.7866601069861274e+09 _runtime‚1455.516140019 _step‚441ç—hÊ˜Ä % lr-AdamW/pg1‚0.001271848261901929 % lr-AdamW/pg2‚0.001271848261901929 trainer/global_step‚5399 % _timestamp‚1.7866601069861274e+09 _runtime‚1455.516140019 _step‚441¹hœ2™- _wandbruntime‚1455  _runtime‚1455wÈà½'š·O N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 5494538240 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 125829120 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚18121702 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17104896 IDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 1379247255040 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚147233388232704 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚16970554 UHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚66779136 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 831258624 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 4628521206 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚146365068836864 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚262 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚262 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 trainer/global_step‚5399 F>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚147233405337600 C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚4289 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚12435819 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 IBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚60 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17104896 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 125829120 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 % _timestamp‚1.7866601069870842e+09 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 122905600 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚41945088 D;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚5264 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚18117415 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚322 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚262 M@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚59049984 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚41945088 JBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚665 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚61478912 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 5368709120 F?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚60 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚4287 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚19976297 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 5368709120 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚18121702 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚223 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚322 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚41348020 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 1304396725760 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚16970475 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚209 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 845178880 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 1301679169582 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 4654602240 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 1304438670848 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚147666806393826 G>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4876 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚148537844008448 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 5494538240 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 5494538240 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚18117415 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚19976299 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚129739508346368 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚147666748006446 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚322 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚148537784958464 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚4534656 _step‚442 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 4654602240 IBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚79 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚20643840 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 122306526 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 125829120 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚322 @7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚5264 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚19976299 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚4287 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚148537844008448 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4289*R蛼O GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚4287 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚148537844008448 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4289 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 5494538240 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 125829120 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚18121702 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17104896 IDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 1379247255040 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚147233388232704 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚16970554 UHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚66779136 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 831258624 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 4628521206 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚146365068836864 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚262 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚262 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 trainer/global_step‚5399 F>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚147233405337600 C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚4289 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚12435819 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 IBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚60 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17104896 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 125829120 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 % _timestamp‚1.7866601069870842e+09 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 122905600 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚41945088 D;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚5264 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚18117415 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚322 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚262 M@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚59049984 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚41945088 JBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚665 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚61478912 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 5368709120 F?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚60 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚4287 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚19976297 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 5368709120 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚18121702 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚223 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚322 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚41348020 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 1304396725760 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚16970475 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚209 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 845178880 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 1301679169582 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 4654602240 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 1304438670848 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚147666806393826 G>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4876 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚148537844008448 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 5494538240 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 5494538240 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚18117415 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚19976299 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚129739508346368 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚147666748006446 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚322 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚148537784958464 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚4534656 _step‚442 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 4654602240 IBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚79 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚20643840 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 122306526 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 125829120 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚322 @7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚5264 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚19976299 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0ºŒÃC2œ- _wandbruntime‚1455  _runtime‚1455­[rÅ&¿M ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 1379441145344 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚38104847 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 1304669742592 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 122905600 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚322 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚19979852 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚19979852 H@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚665 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 1379493569024 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 5494538240 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 >5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5264 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚147259727675392 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 4517085184 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 5368709120 NADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚59053056 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 1301909742194 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 1304669742592 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚18124995 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚16973557 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 1304627794432 $ _timestamp‚1.786660107150172e+09 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚38100552 B9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚5264 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚129762728965632 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 831258624 trainer/global_step‚5399 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚41948160 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚128383308464128 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚147693164372594 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚41348068 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚148564338364928 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚147259727675392 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 1301951090262 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 5494538240 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚262 NGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚75 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 125829120 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 4552982528 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 125829120 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚4295 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚18120702 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 5494538240 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚322 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚147259710570496 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 122306526 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚16973480 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17104896 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚147259710570496 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17104896 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚4535531 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚18124995 EDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚38100552 I@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚4876 D=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚60 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 4552982528 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚38104847 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚19979850 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚223 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 845178880 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚147693222760022 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚146391254630400 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚4293 K>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59053056 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 4654602240 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 122905600 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚129762802033152 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚20643840 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚61478912 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚146391271669760 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 125829120 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚322 IBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚60 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 EDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521206 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚322 DDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚128383287820288 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚262 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚4293 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 1304627794432 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 NADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚58387428 _step‚443 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚148564397417984 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚12438024 SFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚73067520 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 G@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚60 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654602240 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚4535456 _runtime‚ 1455.68045599 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚148564397417984eê¹’Ê&žÄM REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚41948160 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚128383308464128 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚147693164372594 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚41348068 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚148564338364928 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚147259727675392 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 1301951090262 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 5494538240 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚262 NGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚75 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 125829120 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 4552982528 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 125829120 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚4295 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚18120702 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 5494538240 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚322 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚147259710570496 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 122306526 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚16973480 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17104896 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚147259710570496 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17104896 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚4535531 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚18124995 EDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚38100552 I@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚4876 D=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚60 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 4552982528 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚38104847 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚19979850 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚223 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 845178880 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚147693222760022 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚146391254630400 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚4293 K>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59053056 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 4654602240 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 122905600 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚129762802033152 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚20643840 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚61478912 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚146391271669760 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 125829120 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚322 IBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚60 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 EDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521206 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚322 DDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚128383287820288 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚262 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚4293 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 1304627794432 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 NADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚58387428 _step‚443 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚148564397417984 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚12438024 SFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚73067520 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 G@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚60 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654602240 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚4535456 _runtime‚ 1455.68045599 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚148564397417984 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 1379441145344 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚38104847 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 1304669742592 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 122905600 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚322 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚19979852 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚19979852 H@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚665 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 1379493569024 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 5494538240 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 >5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5264 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚147259727675392 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 4517085184 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 5368709120 NADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚59053056 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 1301909742194 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 1304669742592 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚18124995 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚16973557 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 1304627794432 $ _timestamp‚1.786660107150172e+09 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚38100552 B9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚5264 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚129762728965632 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 831258624 trainer/global_step‚5399»øøZ 2Ÿ- _wandbruntime‚1455  _runtime‚1455s 7Er ì 0loss/FRt1/assignment_loss‚0.6415025120347423 %loss/total_loss‚2.295395600515081 trainer/global_step‚5399 % _timestamp‚1.7866601071511016e+09 _step‚444 0loss/FRt2/assignment_loss‚0.6079410181170272 /loss/FRt1/detection_loss‚0.5574399240896077 /loss/FRt2/detection_loss‚0.4885121462737036 epoch‚13 _runtime‚1455.681001971B{Ûw¡ñ /loss/FRt2/detection_loss‚0.4885121462737036 epoch‚13 _runtime‚1455.681001971 0loss/FRt1/assignment_loss‚0.6415025120347423 %loss/total_loss‚2.295395600515081 trainer/global_step‚5399 % _timestamp‚1.7866601071511016e+09 _step‚444 0loss/FRt2/assignment_loss‚0.6079410181170272 /loss/FRt1/detection_loss‚0.5574399240896077¼ŒÅt2¢- _wandbruntime‚1468  _runtime‚1468ç <÷Ç£Á & lr-AdamW/pg2‚0.0012675144371370071 trainer/global_step‚5449 % _timestamp‚1.7866601198599818e+09 _runtime‚1468.390028586 _step‚445 & lr-AdamW/pg1‚0.0012675144371370071ª2n<Ì¤Æ & lr-AdamW/pg1‚0.0012675144371370071 & lr-AdamW/pg2‚0.0012675144371370071 trainer/global_step‚5449 % _timestamp‚1.7866601198599818e+09 _runtime‚1468.390028586 _step‚445½w‡2¥- _wandbruntime‚1468  _runtime‚1468ÒŠW¦ ¦·O TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚41945088 JBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚665 IBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚64 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚262 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 4654602240 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 KDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚60 _runtime‚1468.394907963 C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚4289 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚20153947 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 5368709120 IBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚60 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚38440753 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚18286804 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 1391487127552 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚322 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚429 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚39843840 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 5494538240 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 125829120 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚4287 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚148549505122304 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 5494538240 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 KBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚4876 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 BDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4876 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚148987657881234 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚147674375553024 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚41348020 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17104896 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 122905600 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚149865547663872 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚41945088 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚12546071 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚12546069 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 @7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚5264 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚148549505122304 D;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚5264 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚223 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚4575497 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚130904156702720 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚147674358513664 trainer/global_step‚5449 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚129512609087488 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 122306526 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚130904096215040 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 845178880 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 5368709120 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 125829120 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚149865547663872 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚149865488613888 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4289 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚209 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚148549522227200 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚18286804 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 4654602240 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 1315983491584 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 1316025436672 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚18282517 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 4552982528 PCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚59049984 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚20643840 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚148987599493854 % _timestamp‚1.7866601198609364e+09 F>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚262 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚17121568 PIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚62 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚322 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17104896 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 54945382402Â'§¼O E8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚38436464 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚20153949 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚18282517 G>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4876 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚148987657881234 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚147674375553024 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚41348020 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17104896 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 122905600 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚149865547663872 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚41945088 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚12546071 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚12546069 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 @7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚5264 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚148549505122304 D;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚5264 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚223 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚4575497 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚130904156702720 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚147674358513664 trainer/global_step‚5449 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚129512609087488 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 122306526 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚130904096215040 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 845178880 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 5368709120 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 125829120 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚149865547663872 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚149865488613888 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4289 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚209 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚148549522227200 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚18286804 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 4654602240 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 1315983491584 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 1316025436672 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚18282517 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 4552982528 PCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚59049984 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚20643840 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚148987599493854 % _timestamp‚1.7866601198609364e+09 F>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚262 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚17121568 PIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚62 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚322 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17104896 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 5494538240 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚41945088 JBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚665 IBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚64 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚262 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 4654602240 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 KDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚60 _runtime‚1468.394907963 C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚4289 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚20153947 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 5368709120 IBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚60 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚38440753 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚18286804 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 1391487127552 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚322 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚429 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚39843840 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 5494538240 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 125829120 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚4287 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚148549505122304 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 5494538240 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 KBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚4876 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 B6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚322 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 D=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚60 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚4295 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚38443304 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17104896 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 DDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654602240 H@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚665 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚130927385800704 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚147700544307200 _step‚447 EDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚17124485 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚223 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 NADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚58387428 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 4552982528 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚148575827460096 NGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚65 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 831258624 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚20643840 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚4293 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 1316214560256 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚44035072 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 1316214560256 E6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚322 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 1391773375488 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 5368709120 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 125829120 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 122905600 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 5494538240 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 5368709120 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚322 % _timestamp‚1.7866601200301282e+09 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚38447599 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚4576276 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 5368709120 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 125829120 trainer/global_step‚5449 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚20157500 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 1313471552802 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 5494538240 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚262 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 4654602240 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 1316256508416 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚12548274 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 1313512900870 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚41948160 >5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5264 K>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59053056 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚41348068 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚38447599 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚149892101073408 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚429 SFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚64678912 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚149014074247430 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚149014015860002 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521206 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17104896 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚4293 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚209 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚322 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚262 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚148575844564992 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚148575844564992 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚4576211 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚129535677104128 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚18285804 PDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚38447599 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚4576276 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 5368709120 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 125829120 trainer/global_step‚5449 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚20157500 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 1313471552802 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 5494538240 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚262 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 4654602240 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 1316256508416 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚12548274 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 1313512900870 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚41948160 >5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5264 K>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59053056 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚41348068 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚38447599 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚149892101073408 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚429 SFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚64678912 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚149014074247430 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚149014015860002 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521206 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17104896 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚4293 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚209 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚322 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚262 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚148575844564992 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚148575844564992 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚4576211 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚129535677104128 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚18285804 P6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚322 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 D=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚60 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚4295 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚38443304 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17104896 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 DDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654602240 H@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚665 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚130927385800704 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚147700544307200 _step‚447 EDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚17124485 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚223 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 NADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚58387428 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 4552982528 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚148575827460096 NGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚65 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 831258624 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚20643840 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚4293 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 1316214560256 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚44035072 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 1316214560256 E6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚322 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 1391773375488 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 5368709120¿‡Ë³2«- _wandbruntime‚1468  _runtime‚1468ëܱ“q¬ë 0loss/FRt2/assignment_loss‚0.6354253819630328 /loss/FRt1/detection_loss‚0.5523636346630824 trainer/global_step‚5449 $ _timestamp‚1.786660120031251e+09 _runtime‚1468.561044857 0loss/FRt1/assignment_loss‚0.7895889151145994 /loss/FRt2/detection_loss‚0.4683426487533576 %loss/total_loss‚2.445720580494072 epoch‚13 _step‚448˜c†v­ð %loss/total_loss‚2.445720580494072 epoch‚13 _step‚448 0loss/FRt2/assignment_loss‚0.6354253819630328 /loss/FRt1/detection_loss‚0.5523636346630824 trainer/global_step‚5449 $ _timestamp‚1.786660120031251e+09 _runtime‚1468.561044857 0loss/FRt1/assignment_loss‚0.7895889151145994 /loss/FRt2/detection_loss‚0.4683426487533576Àò™ ®‚(:• ™ŠùÓ¿›á gpu.0.gpu‚45.0 gpu.0.memory‚7, gpu.0.memoryAllocated‚7.0313262939453125) gpu.0.memoryAllocatedBytes‚ 6039863296 gpu.0.temp‚35.0 gpu.0.powerWatts‚132.455' gpu.0.enforcedPowerLimitWatts‚500.0) gpu.0.powerPercent‚26.491000000000003 gpu.0.smClock‚1410 gpu.0.memoryClock‚1593$ gpu.0.correctedMemoryErrors‚1955# gpu.0.uncorrectedMemoryErrors‚0Ô‰x”¯‚(:‰ ™ŠùÓßåòâ cpu‚26.993666019533737 disk.sda3.out‚ 29.39453125 disk.nvme0c0n1.in‚ 17292.8125 disk.sda2.in‚0 disk.sda2.out‚0 disk.dm-1.out‚ 29.39453125" disk.dm-0.out‚52889.1865234375$ memory_percent‚66.07003211975098( proc.memory.availableMB‚ 5559.0859375* disk./.usagePercent‚32.692604759511845 disk.sda3.in‚14134.67578125' disk.nvme0c0n1.out‚52889.1865234375 disk.sda.out‚ 29.39453125% disk.nvme0n1.out‚52889.1865234375 disk.drbd2165.in‚0 disk.drbd2165.out‚0# proc.memory.rssMB‚ 2354.21484375 network.sent‚ 7.267764e+06 network.recv‚ 2.955285e+06 disk.sda1.out‚0 disk.dm-1.in‚14134.67578125* proc.memory.percent‚14.368987083435059 proc.cpu.threads‚17% disk./.usageGB‚487.14343643188477 disk.sda.in‚14134.67578125 disk.sda1.in‚0 disk.nvme0n1.in‚ 17292.8125 disk.dm-0.in‚ 17292.8125ø5& 2°- _wandbruntime‚1481  _runtime‚1481ëÏ¡uƱÀ trainer/global_step‚5499 % _timestamp‚1.7866601327133665e+09 _runtime‚ 1481.24351926 _step‚449 & lr-AdamW/pg1‚0.0012631473733868425 & lr-AdamW/pg2‚0.0012631473733868425óZ˲Š_runtime‚ 1481.24351926 _step‚449 & lr-AdamW/pg1‚0.0012631473733868425 & lr-AdamW/pg2‚0.0012631473733868425 trainer/global_step‚5499 % _timestamp‚1.7866601327133665e+09ÁŒB¿F2³- _wandbruntime‚1481  _runtime‚1481zº ³½'´·O @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚322 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚149865639116800 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚262 KDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚60 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚20331599 PCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚59049984 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚18447619 UHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚68876288 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚20643840 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚130664977727488 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚151193192269312 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17104896 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17104896 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚429 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 IBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚60 F?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚60 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 125829120 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 IDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 PCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚58387380 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 4552982528 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 5494538240 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚12656319 E8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚38779216 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚149865639116800 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4289 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 F>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 4654602240 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚41348020 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 125829120 % _timestamp‚1.7866601327146459e+09 @7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚5264 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚4287 G>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4876 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 1327570257408 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 5368709120 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 _runtime‚1481.245043229 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 122905600 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚38783505 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚17272293 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 PIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚80 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 1324802790798 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚18447619 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 1327612202496 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚20331597 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚132068734939136 D;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚5264 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚209 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 1403805444096 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚322 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚18451906 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚4616054 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚150308509368642 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 122306526 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 5494538240 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚18451906 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 122905600 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚17272375 trainer/global_step‚5499 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚322 M@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚59049984 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 831258624 BDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4876 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 1327570257408 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 5368709120 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 _runtime‚1481.245043229 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 122905600 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚38783505 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚17272293 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 PIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚80 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 1324802790798 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚18447619 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 1327612202496 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚20331597 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚132068734939136 D;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚5264 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚209 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 1403805444096 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚322 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚18451906 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚4616054 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚150308509368642 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 122306526 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 5494538240 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚18451906 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 122905600 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚17272375 trainer/global_step‚5499 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚322 M@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚59049984 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 831258624 BDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚151193192269312 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17104896 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17104896 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚429 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 IBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚60 F?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚60 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 125829120 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 IDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 PCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚58387380 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 4552982528 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 5494538240 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚12656319 E8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚38779216 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚149865639116800 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4289 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 F>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 4654602240 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚41348020 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 125829120 % _timestamp‚1.7866601327146459e+09 @7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚5264ÂÛ™2¶- _wandbruntime‚1481  _runtime‚1481&ÕsVê·ÁM >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚322 trainer/global_step‚5499 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚149891961454592 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚38786056 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚130688025100288 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 5494538240 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚322 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚4293 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 1327843274240 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 1325033363410 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 5494538240 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 125829120 H@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚665 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚150334925734838 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 1325074711478 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 125829120 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 122905600 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚12658524 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654602240 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 5368709120 % _timestamp‚1.7866601328859286e+09 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚38790351 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17104896 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 4552982528 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚18450906 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚223 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521206 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 4552982528 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚151219804728832 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚4295 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚38786056 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 G@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚74 D=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚60 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚41348068 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 5368709120 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 1327801326080 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚20335152 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚4616794 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚132092097059840 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 831258624 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚429 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚150334867347410 G@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚60 K>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59053056 DDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚4293 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚262 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚149891944349696 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚17275392 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚20643840 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚149009833984000 EDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚17275318 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 122905600 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚18455199 NGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚72 PDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚61478912 BDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚151219745675776 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚41948160 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 125829120 >5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5264 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 4654602240 B9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚5264 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 5494538240 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚262 _step‚451 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚322 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 4517085184 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚20335152 NADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚59053056 SFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚66776064 _runtime‚1481.416281141 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚20335150 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚18450906 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚41948160 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17104896 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚4616866 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 1404005183488 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚151219804728832 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚149891961454592 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 1327801326080 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚38790351 NADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚58387428 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚149009851023360¹˜AÌ&¸ÆM E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 1327843274240 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚130688045744128 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 EDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚61478912 BDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚151219745675776 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚41948160 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 125829120 >5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5264 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 4654602240 B9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚5264 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 5494538240 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚262 _step‚451 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚322 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 4517085184 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚20335152 NADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚59053056 SFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚66776064 _runtime‚1481.416281141 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚20335150 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚18450906 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚41948160 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17104896 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚4616866 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 1404005183488 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚151219804728832 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚149891961454592 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 1327801326080 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚38790351 NADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚58387428 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚149009851023360 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚322 trainer/global_step‚5499 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚149891961454592 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚38786056 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚130688025100288 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 5494538240 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚322 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚4293 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 1327843274240 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 1325033363410 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 5494538240 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 125829120 H@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚665 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚150334925734838 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 1325074711478 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 125829120 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 122905600 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚12658524 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654602240 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 5368709120 % _timestamp‚1.7866601328859286e+09 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚38790351 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17104896 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 4552982528 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚18450906 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚223 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521206 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 4552982528 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚151219804728832 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚4295 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚38786056 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 G@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚74 D=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚60 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚41348068 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 5368709120 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 1327801326080 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚20335152 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚4616794 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚132092097059840 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 831258624 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚429 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚150334867347410 G@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚60 K>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59053056 DDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚4293 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚262 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚149891944349696 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚17275392 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚20643840 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚149009833984000 EDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚17275318 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 122905600 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚18455199 NGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚72 PDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚152520895924736 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 4517085184 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚41348020 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚61478912 BDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 122905600 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚152520954974720 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚4287 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 1336405949426 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚262 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚44038144 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚133233398354432 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 1416096025088 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚39126257 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚209 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 PCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚58387380 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 1336364601406 trainer/global_step‚5549 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 122905600 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚4656401 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4289 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚131817346367488 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚151629302468670 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 831258624 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚151629360856050 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚150292937867264 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 5368709120 _step‚454 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚20509249 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 4654602240 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚151181756006400 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 4552982528 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17104896 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 4654602240 M@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚59049984 JBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚665 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 845178880 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 5494538240 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚322 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚322 E8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚39121968 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17104896 IDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4876 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 1339198968320 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚18617008 _runtime‚1494.350908971 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 IDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4876 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 1339198968320 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚18617008 _runtime‚1494.350908971 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 IDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚152520895924736 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 4517085184 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚41348020 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_byƒ$Ätes.small_pool.peak‚61478912 BDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 122905600 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚152520954974720 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚4287 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 1336405949426 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚262 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚44038144 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚133233398354432 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 1416096025088 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚39126257 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚209 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 PCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚58387380 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 1336364601406 trainer/global_step‚5549 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 122905600 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚4656401 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4289 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚131817346367488 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚151629302468670 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 831258624 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚151629360856050 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚150292937867264 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 5368709120 _step‚454 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚20509249 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 4654602240 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚151181756006400 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 4552982528 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17104896 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 4654602240 M@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚59049984 JBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚665 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 845178880 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 5494538240 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚322 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚322 E8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚39121968 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17104896 IDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521206 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 1339388091904 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚39133103 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 1339430040064 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚4657185 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚262 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 _runtime‚1494.518143963 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚39128808 DDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 >5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5264 PDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 % _timestamp‚1.7866601459878075e+09 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚17425897 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚150319140700160 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚46132224 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚151655718834818 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 G@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚60 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚20512800 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 122905600 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚152547508384256 NADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚59053056 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚322 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚151208061239296 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚131840414384128 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚152547508384256 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚41948160 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 831258624 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 EDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚39133103 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚4293 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚17425961 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚322 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚18620301 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17104896 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚20512802 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 trainer/global_step‚5549 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654602240 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚262 D=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚60 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚18620301 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 1416296493568 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 5368709120 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17104896 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 K>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59053056 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚151208078344192 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚131840393740288 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 1336595174018 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 5368709120 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚4295 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 4552982528 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚4657123 SFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚66776064 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚61478912 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 4517085184 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚322 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 4654602240 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚151655777222246 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 5494538240 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚151208061239296 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚41948160 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 122905600 H@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚665 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 122306526 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 5494538240 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚262 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚429 NGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚62 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚133256690233856 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚152547449331200 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚12768774 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚223 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚322 BDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 G@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚60 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚20512800 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 122905600 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚152547508384256 NADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚59053056 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚322 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚151208061239296 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚131840414384128 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚152547508384256 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚41948160 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 831258624 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 EDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚39133103 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚4293 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚17425961 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚322 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚18620301 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17104896 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚20512802 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 trainer/global_step‚5549 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654602240 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚262 D=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚60 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚18620301 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 1416296493568 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 5368709120 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17104896 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 K>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59053056 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚151208078344192 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚131840393740288 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 1336595174018 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 5368709120 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚4295 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 4552982528 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚4657123 SFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚66776064 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚61478912 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 4517085184 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚322 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 4654602240 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚151655777222246 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 5494538240 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚151208061239296 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚41948160 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 122905600 H@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚665 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 122306526 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 5494538240 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚262 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚429 NGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚62 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚133256690233856 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚152547449331200 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚12768774 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚223 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚322 BDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521206 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 1339388091904 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚39133103 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 1339430040064 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚4657185 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚262 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 _runtime‚1494.518143963 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚39128808 DDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 >5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5264 PDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 % _timestamp‚1.7866601459878075e+09 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚17425897 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚150319140700160 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚46132224ǽ{°¶2Ç- _wandbruntime‚1494  _runtime‚1494âT¨rÈì /loss/FRt1/detection_loss‚0.5033638539234292 /loss/FRt2/detection_loss‚0.4890122564138405 epoch‚13 trainer/global_step‚5549 $ _timestamp‚1.786660145988872e+09 _runtime‚1494.518653245 _step‚456 0loss/FRt1/assignment_loss‚0.7070301984560465 0loss/FRt2/assignment_loss‚0.6371081667536902 &loss/total_loss‚2.3365144755470064E-ñàwÉñ /loss/FRt1/detection_loss‚0.5033638539234292 /loss/FRt2/detection_loss‚0.4890122564138405 epoch‚13 trainer/global_step‚5549 $ _timestamp‚1.786660145988872e+09 _runtime‚1494.518653245 _step‚456 0loss/FRt1/assignment_loss‚0.7070301984560465 0loss/FRt2/assignment_loss‚0.6371081667536902 &loss/total_loss‚2.3365144755470064ÈÀa4‰–Ê‚(:‹ ·ŠùÓªŽ¦á gpu.0.gpu‚55.0 gpu.0.memory‚11, gpu.0.memoryAllocated‚7.0313262939453125) gpu.0.memoryAllocatedBytes‚ 6039863296 gpu.0.temp‚35.0 gpu.0.powerWatts‚225.671' gpu.0.enforcedPowerLimitWatts‚500.0 gpu.0.powerPercent‚45.1342 gpu.0.smClock‚1410 gpu.0.memoryClock‚1593$ gpu.0.correctedMemoryErrors‚1955# gpu.0.uncorrectedMemoryErrors‚0ö:È7™Ë‚(:Ž ·ŠùÓ²Øçã network.sent‚ 7.432613e+06% disk./.usageGB‚487.15921783447266 disk.sda2.in‚0" disk.nvme0n1.in‚17337.95703125 network.recv‚ 3.014104e+06' disk./.usagePercent‚32.693663862681 disk.dm-1.in‚14134.67578125 disk.drbd2165.out‚0 disk.dm-0.in‚17337.95703125# disk.dm-0.out‚52918.94677734375# proc.memory.rssMB‚ 2354.21484375 cpu‚26.994407162423883$ disk.nvme0c0n1.in‚17337.95703125( disk.nvme0c0n1.out‚52918.94677734375 disk.sda.out‚ 30.140625 disk.sda1.in‚0 disk.sda2.out‚0& disk.nvme0n1.out‚52918.94677734375 disk.dm-1.out‚ 30.140625 disk.drbd2165.in‚0 disk.sda3.in‚14134.67578125 disk.sda3.out‚ 30.140625 disk.sda.in‚14134.67578125 disk.sda1.out‚0# memory_percent‚66.6374683380127( proc.memory.availableMB‚ 5466.1171875* proc.memory.percent‚14.368987083435059 proc.cpu.threads‚17ñBn2Ì- _wandbruntime‚1507  _runtime‚1507ï¢Î™ÇÍÁ & lr-AdamW/pg1‚0.0012543146530010244 & lr-AdamW/pg2‚0.0012543146530010244 trainer/global_step‚5599 % _timestamp‚1.7866601593115726e+09 _runtime‚1507.841648949 _step‚457øMÊÌÎÆ & lr-AdamW/pg2‚0.0012543146530010244 trainer/global_step‚5599 % _timestamp‚1.7866601593115726e+09 _runtime‚1507.841648949 _step‚457 & lr-AdamW/pg1‚0.0012543146530010244É…5÷\2Ï- _wandbruntime‚1507  _runtime‚1507óÆÂ7ÔжO M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚17573457 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚152950153956078 M@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚59049984 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 4654602240 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚4696638 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 UHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚70973440 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 F?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚60 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚41945088 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 4517085184 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚41945088 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚132969735651328 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚151602227544064 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 E8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚39464720 PIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚68 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 5494538240 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚262 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚152497872896000 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 IDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 5368709120 _step‚458 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚18782110 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚152497855791104 KBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚4876 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚50329600 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 1347926412014 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 5368709120 PCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚59049984 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 4654602240 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚41348020 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 4552982528 D;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚5264 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚17573527 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚4696706 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 125829120 IDev¸¯ûèiceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚153848599580160 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 1350785734144 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚134398087987200 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚4289 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 1428352335872 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚262 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚152497872896000 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4289 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 4628521206 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚151602244583424 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 122306526 trainer/global_step‚5599 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚20686899 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17104896 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚18782110 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚12876821 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 845178880 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 125829120 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚152497855791104 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 1350785734144 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚153848599580160 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚134398017013760 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚322 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 122905600 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚153848658630144 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚209 @7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚5264 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚132969715007488 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 1347967760034 _runtime‚1507.842976116 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 1350743789056 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚20686899 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 5494538240 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚322 G>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4876 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚12876819 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚20643840 IBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚60 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚20686897 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 831258624 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚152950212343458 F>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚18777823 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 KDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚60 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚39469009 JBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚665 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚223 BDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4876 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚12876819 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚20643840 IBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚60 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚20686897 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 831258624 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚152950212343458 F>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚18777823 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 KDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚60 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚39469009 JBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚665 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚223 BDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 5368709120 _step‚458 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚18782110 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚152497855791104 KBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚4876 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚50329600 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 1347926412014 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 5368709120 PCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚59049984 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 4654602240 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚41348020 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 4552982528 D;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚5264 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚17573527 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚4696706 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 125829120 IDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚153848599580160 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 1350785734144 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚134398087987200 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚4289 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 1428352335872 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚262 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚152497872896000 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4289 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 4628521206 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚151602244583424 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 122306526 trainer/global_step‚5599 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚20686899 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17104896 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚18782110 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚12876821 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 845178880 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 125829120 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚152497855791104 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 1350785734144 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚153848599580160 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚134398017013760 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚322 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 122905600 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚153848658630144 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚209ÊÊp$V2Ò- _wandbruntime‚1508  _runtime‚1508 §wÇ&ÓÁM \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚134421374292992 BDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 G@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚69 D=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚60 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 4552982528 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚39475855 H@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚665 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 845178880 trainer/global_step‚5599 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚12879026 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚41348068 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 5368709120 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚18781110 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚223 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚132992762380288 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚20690450 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 1350974857728 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521206 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚39475855 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚41948160 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 1428547233792 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 DDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 IBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚60 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654602240 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚4295 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 4654602240 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚152524178128896 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚262 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚4293 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚153875152986624 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 5368709120 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 125829120 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚134421309614080 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚262 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚17576514 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚322 NADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚58387428 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚20690452 EDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59053056 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚41948160 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 1351016805888 B9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚5264 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚39471560 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚20690452 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚4697488 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 % _timestamp‚1.7866601594826918e+09 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚18785403 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚153875212039680 NADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚59053056 E6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚322 >5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5264 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17104896 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚17576445 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2=Â'L;ÔÆM PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 122905600 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚44035072 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 K>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59053056 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚41948160 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 1351016805888 B9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚5264 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚39471560 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚20690452 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚4697488 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 % _timestamp‚1.7866601594826918e+09 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚18785403 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚153875212039680 NADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚59053056 E6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚322 >5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5264 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17104896 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚17576445 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚134421374292992 BDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 G@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚69 D=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚60 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 4552982528 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚39475855 H@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚665 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 845178880 trainer/global_step‚5599 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚12879026 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚41348068 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 5368709120 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚18781110 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚223 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚132992762380288 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚20690450 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 1350974857728 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521206 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚39475855 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚41948160 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 1428547233792 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 DDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 IBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚60 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654602240 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚4295 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 4654602240 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚152524178128896 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚262 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚4293 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚153875152986624 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 5368709120 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 125829120 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚134421309614080 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚262 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚17576514 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚322 NADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚58387428 NADeviceStatsMonitor.on_train_batch_end/activeŒ¦‘.large_pool.allocated‚20690452 EDeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4289 IDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚41348020 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 125829120 IBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚60 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚18969404 PCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚59049984 JBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚665 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚223 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 845178880 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 4517085184 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 5494538240 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚262 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚17774676 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚322 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚262 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚262 @7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚5264 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚155571308176896 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 1366398281216 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚44038144 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 trainer/global_step‚5649 G>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4876 PIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚67 UHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚64681984 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 125829120 F>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 122905600 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚20925153 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 4628521206 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 5368709120 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚322 _step‚463 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 4552982528 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚18969404 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚322 BDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚41348020 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 125829120 IBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚60 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚18969404 PCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚59049984 JBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚665 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚223 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 845178880 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 4517085184 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 5494538240 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚262 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚17774676 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚322 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚262 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚262 @7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚5264 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚155571308176896 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 1366398281216 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚44038144 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 trainer/global_step‚5649 G>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4876 PIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚67 UHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚64681984 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 125829120 F>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 122905600 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚20925153 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 4628521206 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 5368709120 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚322 _step‚463 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 4552982528 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚18969404 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚322 BDeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4289 I6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚322 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚262 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 122905600 SFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚66776064 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 1363784767340 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 122306526 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 NGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚58 BDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚39901405 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚41348068 IBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚60 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 K>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59053056 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚154682445661008 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚154231249338368 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚154231232233472 I@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚4876 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚61478912 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚39897110 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 4654602240 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚155597920636416 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚135889295399424 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 4517085184 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 1363826115408 >5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5264 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚18968404 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚154682387273580 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521206 JBDevicõÁ°¡›eStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚262 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 1366629349888 B9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚5264 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚20928706 H@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚665 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚209 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚153318619545600 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚135889362175488 DDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚4747727 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 5368709120 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚322 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚39901405 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚18968404 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 4552982528 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚41948160 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚4747785 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚20928708 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚17777615 E6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚322 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚153318602506240 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 1366671298048 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚223 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 1444894071296 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚262 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654602240qRZ¢Ë&åÅM NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚18968404 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚154682387273580 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521206 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚262 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 1366629349888 B9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚5264 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚20928706 H@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚665 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚209 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚153318619545600 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚135889362175488 DDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚4747727 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 5368709120 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚322 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚39901405 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚18968404 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 4552982528 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚41948160 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚4747785 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚20928708 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚17777615 E6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚322 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚153318602506240 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 1366671298048 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚223 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 1444894071296 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚262 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654602240 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 D=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚60 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 5494538240 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚322 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚262 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 122905600 SFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚66776064 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 1363784767340 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 122306526 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 NGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚58 BDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚39901405 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚41348068 IBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚60 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 K>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59053056 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚154682445661008 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚154231249338368 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚154231232233472 I@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚4876 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚61478912 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚39897110 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 4654602240 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚155597920636416 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚135889295399424 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 4517085184 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 1363826115408 >5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5264СÔ\2æ- _wandbruntime‚1526  _runtime‚1526ce²rçì _runtime‚1526.719106823 _step‚465 0loss/FRt1/assignment_loss‚0.7207022038562076 0loss/FRt2/assignment_loss‚0.6757430836972739 epoch‚14 trainer/global_step‚5649 % _timestamp‚1.7866601781893008e+09 /loss/FRt1/detection_loss‚0.5355914437754609 /loss/FRt2/detection_loss‚0.5116832413007992 %loss/total_loss‚2.443719972629742$ùrwèñ _step‚465 0loss/FRt1/assignment_loss‚0.7207022038562076 0loss/FRt2/assignment_loss‚0.6757430836972739 epoch‚14 trainer/global_step‚5649 % _timestamp‚1.7866601781893008e+09 /loss/FRt1/detection_loss‚0.5355914437754609 /loss/FRt2/detection_loss‚0.5116832413007992 %loss/total_loss‚2.443719972629742 _runtime‚1526.719106823ÑéÈ;”é‚(:‰ ÕŠùӛ̆á gpu.0.gpu‚0.0 gpu.0.memory‚0, gpu.0.memoryAllocated‚7.0313262939453125) gpu.0.memoryAllocatedBytes‚ 6039863296 gpu.0.temp‚34.0 gpu.0.powerWatts‚120.113' gpu.0.enforcedPowerLimitWatts‚500.0 gpu.0.powerPercent‚24.0226 gpu.0.smClock‚1410 gpu.0.memoryClock‚1593$ gpu.0.correctedMemoryErrors‚1955# gpu.0.uncorrectedMemoryErrors‚0MDà…—ê‚(:Œ ÕŠùÓûÊÞã disk.sda1.in‚0 disk.sda1.out‚0 disk.dm-0.in‚ 17338.015625# disk.dm-0.out‚52989.52099609375* proc.memory.percent‚14.506864547729492 network.recv‚ 3.075158e+06$ disk./.usageGB‚487.1622123718262 disk.sda3.out‚ 30.6953125" disk.nvme0c0n1.in‚ 17338.015625 disk.dm-1.out‚ 30.6953125 disk.drbd2165.in‚0$ memory_percent‚66.77520275115967) proc.memory.availableMB‚ 5443.55078125 network.sent‚ 7.579652e+06 disk.sda3.in‚14134.67578125 disk.sda.in‚14134.67578125 disk.sda.out‚ 30.6953125& disk.nvme0n1.out‚52989.52099609375 disk.dm-1.in‚14134.67578125 cpu‚27.01062965416184) disk./.usagePercent‚32.69386482859537( disk.nvme0c0n1.out‚52989.52099609375 disk.sda2.in‚0 disk.sda2.out‚0 disk.nvme0n1.in‚ 17338.015625 disk.drbd2165.out‚0" proc.memory.rssMB‚ 2376.8046875 proc.cpu.threads‚17ñŠyH2ë- _wandbruntime‚1539  _runtime‚1539I™±ÂÇìÁ & lr-AdamW/pg1‚0.0012453523699445378 & lr-AdamW/pg2‚0.0012453523699445378 trainer/global_step‚5699 % _timestamp‚1.7866601912610886e+09 _runtime‚1539.791152917 _step‚466N¸BàÌíÆ & lr-AdamW/pg1‚0.0012453523699445378 & lr-AdamW/pg2‚0.0012453523699445378 trainer/global_step‚5699 % _timestamp‚1.7866601912610886e+09 _runtime‚1539.791152917 _step‚466ÒmÒ—2î- _wandbruntime‚1539  _runtime‚15395#ܽ'ï·O IDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4876 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚17926020 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 1456979003392 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚262 KDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚60 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚4287 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 PCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚58387380 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 1375157353356 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚322 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚223 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚4788163 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚322 _step‚467 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 1377985047040 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 5368709120 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚262 UHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚70973440 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚322 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 5494538240 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 125829120 IDeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4289 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 122306526 PCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚59049984 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚156899011832320 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚4788087 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 4517085184 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 F?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚60 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚19130219 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17104896 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚41945088 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚154601706389504 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 125829120 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 4654602240 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 122905600 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚209 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 5494538240 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 % _timestamp‚1.7866601912620974e+09 E8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚40233022 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚156899011832320 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚155521043890176 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 1378026992128 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚21102803 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚135573743239168 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚21102805 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚20643840 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚19134506 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 5494538240 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚21102803 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚17926098 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 845178880 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚135573722595328 _runtime‚1539.795887641 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17104896 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚50329600 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 4654602240 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚13137933 F>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 JBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚665 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚429 PIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚76 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 1378026992128 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 125829120 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 4552982528 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 1377985047040 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚13137935 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 4628521206 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚41348020 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 4552982528 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 1456928673792 BDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4876 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚17926020 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 1456979003392 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚262 KDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚60 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚4287 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 PCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚58387380 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 1375157353356 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚322 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚223 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚4788163 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚322 _step‚467 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 1377985047040 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 5368709120 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚262 UHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚70973440 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚322 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 5494538240 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 125829120 IDeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4289 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 122306526 PCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚59049984 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚156899011832320 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚4788087 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 4517085184 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 F?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚60 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚19130219 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17104896 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚41945088 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚154601706389504 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 125829120 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 4654602240 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 122905600 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚209 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 5494538240 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.al׌] }located‚0 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 % _timestamp‚1.7866601912620974e+09 E8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚40233022 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚156899011832320 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚155521043890176 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 1378026992128 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚21102803 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚135573743239168 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚21102805 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚20643840 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚19134506 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 5494538240 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚21102803 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚17926098 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 845178880 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚135573722595328 _runtime‚1539.795887641 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17104896 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚50329600 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 4654602240 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚13137933 F>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 JBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚665 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚429 PIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚76 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 1378026992128 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 125829120 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 4552982528 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 1377985047040 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚13137935 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 4628521206 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚41348020 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 4552982528 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 1456928673792 B8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚322 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚17929138 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 1378258063872 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 5368709120 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚156925565238784 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 122306526 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 K>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59053056 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚40244157 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚13140138 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚262 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚13140140 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 D5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5264 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚156925624291840 G@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚72 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚4788998 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 NADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚58387428 BDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚156925624291840 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚155547349123072 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 122905600 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 1378216115712 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚19137799 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚19137799 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚155547366227968 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚41948160 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚21106356 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚209 trainer/global_step‚5699 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚154627892183040 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚262 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚21106356 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚61478912 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521206 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17104896 NADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚59053056 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚20643840 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 _runtime‚1539.965023225 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 4552982528 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚322 _step‚468 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 1378258063872 EDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654602240 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚19133506 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚4295 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚40239862 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚21106358 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚4788928 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 4517085184 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚19133506 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 1457221934080 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚322 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 5368709120 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚40244157 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 4654602240 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 1378216115712 I@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚4876 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚137053945769984 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 4552982528 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17104896 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 845178880 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚135596790611968 D=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚60 % _timestamp‚1.7866601914346054e+09 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚135596769968128 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚156003297148416 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 125829120 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚322 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚154627909222400 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 5368709120 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚429 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚17929066 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 P6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚322 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚154627909222400 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 5368709120 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚429 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚17929066 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 P8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚322 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚17929138 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 1378258063872 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 5368709120 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚156925565238784 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 122306526 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 K>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59053056 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚40244157 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚13140138 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚262 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚13140140 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 D5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5264 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚156925624291840 G@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚72 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚4788998 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 NADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚58387428 BDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚156925624291840 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚155547349123072 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 122905600 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 1378216115712 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚19137799 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚19137799 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚155547366227968 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚41948160 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚21106356 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚209 trainer/global_step‚5699 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚154627892183040 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚262 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚21106356 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚61478912 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521206 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17104896 NADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚59053056 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚20643840 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 _runtime‚1539.965023225 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 4552982528 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚322 _step‚468 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 1378258063872 EDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654602240 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚19133506 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚4295 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚40239862 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚21106358 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚4788928 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 4517085184 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚19133506 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 1457221934080 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚322 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 5368709120 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚40244157 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 4654602240 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 1378216115712 I@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚4876 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚137053945769984ÔiúKk2ô- _wandbruntime‚1539  _runtime‚1539¾‘'7rõì &loss/total_loss‚2.4513705013291296 % _timestamp‚1.7866601914357467e+09 _runtime‚ 1539.96569303 0loss/FRt1/assignment_loss‚0.7249136079353241 0loss/FRt2/assignment_loss‚0.7101251976007398 epoch‚14 trainer/global_step‚5699 _step‚469 /loss/FRt1/detection_loss‚0.5181663860866637 /loss/FRt2/detection_loss‚0.4981653097064021‡¬’­wöñ /loss/FRt1/detection_loss‚0.5181663860866637 /loss/FRt2/detection_loss‚0.4981653097064021 &loss/total_loss‚2.4513705013291296 % _timestamp‚1.7866601914357467e+09 _runtime‚ 1539.96569303 0loss/FRt1/assignment_loss‚0.7249136079353241 0loss/FRt2/assignment_loss‚0.7101251976007398 epoch‚14 trainer/global_step‚5699 _step‚469Õ^S‚ô–÷‚(:‹ äŠùÓÙÜîà gpu.0.gpu‚61.0 gpu.0.memory‚11, gpu.0.memoryAllocated‚7.0313262939453125) gpu.0.memoryAllocatedBytes‚ 6039863296 gpu.0.temp‚35.0 gpu.0.powerWatts‚181.409' gpu.0.enforcedPowerLimitWatts‚500.0 gpu.0.powerPercent‚36.2818 gpu.0.smClock‚1410 gpu.0.memoryClock‚1593$ gpu.0.correctedMemoryErrors‚1955# gpu.0.uncorrectedMemoryErrors‚0D/ÊYø‚(:… äŠùÓ¿éÅã network.recv‚ 3.095326e+06 disk.sda1.in‚0* proc.memory.percent‚14.506864547729492 proc.cpu.threads‚17) disk./.usagePercent‚32.69403686565839 disk.sda3.in‚14180.30078125& disk.nvme0c0n1.out‚52994.666015625 disk.sda2.in‚0$ memory_percent‚66.77336692810059( proc.memory.availableMB‚ 5443.8515625" proc.memory.rssMB‚ 2376.8046875 cpu‚27.01164225666255$ disk./.usageGB‚487.1647758483887$ disk.nvme0c0n1.in‚17348.33984375 disk.sda.in‚14180.30078125 disk.sda1.out‚0 disk.sda2.out‚0$ disk.nvme0n1.out‚52994.666015625 disk.dm-1.out‚30.96875! disk.dm-0.out‚52994.666015625 network.sent‚ 7.643004e+06 disk.sda3.out‚30.96875 disk.sda.out‚30.96875" disk.nvme0n1.in‚17348.33984375 disk.dm-1.in‚14180.30078125 disk.drbd2165.in‚0 disk.drbd2165.out‚0 disk.dm-0.in‚17348.33984375[îÀ2ù- _wandbruntime‚1552  _runtime‚1552Õ]mÆúÀ $ _timestamp‚1.786660204411587e+09 _runtime‚1552.941696199 _step‚470 & lr-AdamW/pg1‚0.0012408233606574996 & lr-AdamW/pg2‚0.0012408233606574996 trainer/global_step‚5749q–¥ÆËûÅ $ _timestamp‚1.786660204411587e+09 _runtime‚1552.941696199 _step‚470 & lr-AdamW/pg1‚0.0012408233606574996 & lr-AdamW/pg2‚0.0012408233606574996 trainer/global_step‚5749ÖÇwkÏ2ü- _wandbruntime‚1552  _runtime‚1552LŒ‡µ&ý¶O L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 4654602240 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚21280455 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 1469199832064 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 122306526 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 4654602240 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚21280453 BDeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4289 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚18077205 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 5368709120 IBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚71 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚41945088 G>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4876 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚19299608 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 4552982528 D;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚5264 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚4829020 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚136726111879168 E8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚40575774 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 1389571812864 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚13248185 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚223 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 5494538240 JBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚665 `LDeviceStatsMonitor.on_train_batch_start/requested_byt,ˆÃß–es.large_pool.allocated‚155911013105664 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚322 IBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚60 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚156837143674880 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚19295321 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚322 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚262 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 trainer/global_step‚5749 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚158226715487744 PIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚69 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 845178880 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚157297732269628 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 _runtime‚ 1552.94590521 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚19295321 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 1389613757952 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚13248183 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 122905600 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚40580063 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚4287 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚21280453 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚156837143674880 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 _step‚471 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚18077134 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚136726091235328 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚262 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚61478912 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 4517085184 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚158226774537728 F>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚20643840 IDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚158226715487744 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17104896 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚19299608 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 PCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚58387380 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚157297673882248 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 1389613757952 @7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚5264ÿ>Z5Á'þ»O UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 5368709120 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 4552982528 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚156837160779776 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚138195291067392 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚21280455 M@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚59049984 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚209 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚44038144 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 125829120 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚322 % _timestamp‚1.7866602044127371e+09 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚138195355749376 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚158226774537728 PCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚59049984 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4289 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚18077205 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 5368709120 IBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚71 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚41945088 G>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4876 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚19299608 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 4552982528 D;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚5264 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚4829020 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚136726111879168 E8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚40575774 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 1389571812864 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚13248185 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚223 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 5494538240 JBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚665 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚155911013105664 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚322 IBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚60 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚156837143674880 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚19295321 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚322 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚262 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 trainer/global_step‚5749 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚158226715487744 PIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚69 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 845178880 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚157297732269628 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 _runtime‚ 1552.94590521 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚19295321 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 1389613757952 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚13248183 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 122905600 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚40580063 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚4287 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚21280453 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚156837143674880 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 _step‚471 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚18077134 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚136726091235328 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚262 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚61478912 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 4517085184 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚158226774537728 F>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚20643840 IDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚158226715487744 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17104896 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚19299608 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 PCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚58387380 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚157297673882248 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 1389613757952 @7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚5264 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 4654602240 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚21280455 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 1469199832064 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 122306526 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 4654602240 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚21280453 BDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚40586909 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚4829765 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 125829120 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚4295 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚21284006 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚138218583634944 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚155937181859840 I@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚4876 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚46132224 >5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5264 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 1469491159040 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 5494538240 P6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚322 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 NADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚59053056 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚19302901 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 5368709120 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 % _timestamp‚1.7866602045809226e+09 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚21284006 NGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚64 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 1386908388556 IBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚60 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚21284008 DDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚18080153 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚138218650411008 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 122306526 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚322 H@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚665 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚262 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚156863466012672 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚41948160 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚40582614 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚13250390 BDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521206 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚41348068 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚40582614 K>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59053056 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚157324090248396 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 125829120 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚19302901 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚158253268894208 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 1389844829696 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚4293 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚19298608 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 122905600 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚158253327947264 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654602240 G@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚66 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚223 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚209 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚158253327947264 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 1389802881536 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚21284008 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚136749159251968 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚20643840 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚155937198899200 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 125829120 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚156863483117568 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚4829829 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 5494538240 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚156863483117568 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚429 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 4517085184 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 NADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚58387428 G@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚60 _runtime‚1553.111367644 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 1389802881536 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚18080219 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚157324148635824 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 4654602240 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17104896 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 4552982528ŸOd†ÆM QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 4552982528 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚136749138608128 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 5368709120 EDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚40586909 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚4829765 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 125829120 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚4295 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚21284006 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚138218583634944 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚155937181859840 I@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚4876 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚46132224 >5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5264 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 1469491159040 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 5494538240 P6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚322 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 NADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚59053056 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚19302901 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 5368709120 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 % _timestamp‚1.7866602045809226e+09 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚21284006 NGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚64 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 1386908388556 IBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚60 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚21284008 DDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚18080153 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚138218650411008 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 122306526 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚322 H@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚665 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚262 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚156863466012672 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚41948160 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚40582614 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚13250390 BDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521206 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚41348068 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚40582614 K>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59053056 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚157324090248396 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 125829120 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚19302901 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚158253268894208 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 1389844829696 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚4293 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚19298608 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 122905600 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚158253327947264 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654602240 G@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚66 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚223 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚209 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚158253327947264 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 1389802881536 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚21284008 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚1367šÕ? 49159251968 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚20643840 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚155937198899200 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 125829120 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚156863483117568 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚4829829 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 5494538240 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚156863483117568 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚429 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 4517085184 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 NADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚58387428 G@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚60 _runtime‚1553.111367644 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 1389802881536 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚18080219 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚157324148635824 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 4654602240 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17104896 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 4552982528 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 SFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚66776064 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 1386949736624 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚322 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚61478912 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚262 EDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚159554419143168 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚158153260564480 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚18228142 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚158153277669376 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 1401200523776 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚4869707 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 % _timestamp‚1.7866602170639741e+09 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 122905600 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚159554419143168 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚13358433 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚139359925274112 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 5494538240 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚262 KDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚60 KBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚4876 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 845178880 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚20643840 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 831258624 IBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚60 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚4869630 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚158618583757036 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 1398280974572 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚21458103 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚19464710 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚50329600 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚158618525369656 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 JBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚665 IBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚79 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 5494538240 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚21458105 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17104896 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚4287 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚137878480519168 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 1481465398784 IDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 1401158578688 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚21458103 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚19460423 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 1401200523776 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4289 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚157220302782464 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚322 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚19460423 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 5368709120 _runtime‚1565.597878554 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚262 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚19464710 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 4654602240 PIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚77 G>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4876 IDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 1401158578688 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚21458103 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚19460423 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 1401200523776 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4289 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚157220302782464 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚322 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚19460423 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 5368709120 _runtime‚1565.597878554 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚262 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚19464710 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 4654602240 PIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚77 G>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4876 IDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚159554419143168 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚158153260564480 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚18228142 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚158153277669376 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 1401200523776 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚4869707 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 % _timestamp‚1.7866602170639741e+09 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 122905600 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚159554419143168 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚13358433 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚139359925274112 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 5494538240 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚262 KDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚60 KBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚4876 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 845178880 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚20643840 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 831258624 IBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚60 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚4869630 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚158618583757036 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 1398280974572 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚21458103 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚19464710 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚50329600 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚158618525369656 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 JBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚665 IBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚79 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 5494538240 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚21458105 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17104896 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚4287 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚137878480519168 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 1481465398784 I5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5264 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚159580972549632 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 845178880 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 PDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚158179600007168 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚262 IBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚60 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 1401431595520 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚158644941735804 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚322 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚19463710 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚158179582902272 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 4552982528 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚157246471536640 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚262 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 1401389647360 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚223 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521206 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 BDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654602240 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 1481709184512 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 5494538240 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 125829120 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 4654602240 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚13360640 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 125829120 trainer/global_step‚5799 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚139383291597312 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚21461658 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 5494538240 % _timestamp‚1.7866602172363968e+09 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚40929661 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚4295 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚21461658 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 1401431595520 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚322 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚158645000123232 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 G@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚60 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 5368709120 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚158179600007168 SFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚75164672 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚19468003 K>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59053056 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚18231175 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 H@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚665 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚429 B9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚5264 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚21461656 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚21461656 G@DeviceStatsMonitor.on_train_batch_end/inactive_split8S§»Q .all.current‚75 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 1398511547232 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚159581031602688 NGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚73 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚4293 I@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚4876 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 831258624 NADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚58387428 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17104896 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚40929661 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 5368709120 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17104896 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚40925366 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚18231100 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚13360638 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚137901527891968 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 5494538240 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚322 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚41948160 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚262 DDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚18231100 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚13360638 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚137901527891968 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 5494538240 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚322 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚41948160 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚262 D5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5264 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚159580972549632 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 845178880 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 PDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚158179600007168 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚262 IBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚60 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 1401431595520 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚158644941735804 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚322 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚19463710 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚158179582902272 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 4552982528 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚157246471536640 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚262 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 1401389647360 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚223 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521206 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 BDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654602240 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 1481709184512 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 5494538240 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 125829120 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 4654602240 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚13360640 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 125829120 trainer/global_step‚5799 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚139383291597312 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚21461658 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 5494538240 % _timestamp‚1.7866602172363968e+09 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚40929661 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚4295 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚21461658 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 1401431595520 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚322 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚158645000123232 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 G@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚60 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 5368709120 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚158179600007168 SFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚75164672 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚19468003 K>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59053056 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚18231175 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 H@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚665 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚429 B9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚5264 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚21461656 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚21461656 G@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚75 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 1398511547232 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚159581031602688 NGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚73 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚4293 I@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚4876 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 831258624 NADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚58387428 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17104896 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚40929661Üg¿É2- _wandbruntime‚1565  _runtime‚1565â†4jr‘ì trainer/global_step‚5799 _runtime‚1565.767250368 _step‚477 0loss/FRt1/assignment_loss‚0.7735261694806647 &loss/total_loss‚2.4540447561953815 epoch‚14 % _timestamp‚1.7866602172374535e+09 0loss/FRt2/assignment_loss‚0.6758308347223823 /loss/FRt1/detection_loss‚0.5029609838942496 .loss/FRt2/detection_loss‚0.501726768098085üD7w’ñ 0loss/FRt2/assignment_loss‚0.6758308347223823 /loss/FRt1/detection_loss‚0.5029609838942496 .loss/FRt2/detection_loss‚0.501726768098085 trainer/global_step‚5799 _runtime‚1565.767250368 _step‚477 0loss/FRt1/assignment_loss‚0.7735261694806647 &loss/total_loss‚2.4540447561953815 epoch‚14 % _timestamp‚1.7866602172374535e+09Ý+ë» “‚(:• ‚‹ùÓ„Ó¤á gpu.0.gpu‚14.0 gpu.0.memory‚2, gpu.0.memoryAllocated‚7.0313262939453125) gpu.0.memoryAllocatedBytes‚ 6039863296 gpu.0.temp‚35.0 gpu.0.powerWatts‚159.003' gpu.0.enforcedPowerLimitWatts‚500.0) gpu.0.powerPercent‚31.800599999999996 gpu.0.smClock‚1410 gpu.0.memoryClock‚1593$ gpu.0.correctedMemoryErrors‚1955# gpu.0.uncorrectedMemoryErrors‚0€ªáŸ”‚(:” ‚‹ùÓÉü¸ä disk.dm-1.out‚ 31.41015625 disk.drbd2165.in‚0 disk.sda3.in‚15349.44140625( disk.nvme0c0n1.out‚53046.75830078125$ memory_percent‚66.77796840667725" proc.memory.rssMB‚ 2376.8046875 network.sent‚ 7.773133e+06 disk.sda.in‚15349.44140625 disk.sda1.in‚0& disk.nvme0n1.out‚53046.75830078125 disk.drbd2165.out‚0# disk.dm-0.out‚53046.75830078125 proc.cpu.threads‚17 network.recv‚ 3.138628e+06) disk./.usagePercent‚32.69825126168705$ disk./.usageGB‚487.2275733947754 disk.sda2.out‚0" disk.nvme0n1.in‚17375.22265625 disk.dm-0.in‚ 17375.2265625 disk.sda3.out‚ 31.41015625$ disk.nvme0c0n1.in‚17375.22265625 disk.sda1.out‚0 disk.sda2.in‚0) proc.memory.availableMB‚ 5443.09765625* proc.memory.percent‚14.506864547729492 cpu‚27.01325476320574 disk.sda.out‚ 31.41015625 disk.dm-1.in‚15349.44140625E65X2•- _wandbruntime‚1578  _runtime‚1578;ƒôÇ–Á _step‚478 & lr-AdamW/pg1‚0.0012316710609961664 & lr-AdamW/pg2‚0.0012316710609961664 trainer/global_step‚5849 % _timestamp‚1.7866602299124236e+09 _runtime‚1578.442438644uÇÃÌ—Æ _step‚478 & lr-AdamW/pg1‚0.0012316710609961664 & lr-AdamW/pg2‚0.0012316710609961664 trainer/global_step‚5849 % _timestamp‚1.7866602299124236e+09 _runtime‚1578.442438644ÞÇÂy&2˜- _wandbruntime‚1578  _runtime‚1578´¤½'™·O YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 125829120 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 4654602240 JBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚665 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚160882181848576 G>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4876 IDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚160882122798592 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚262 % _timestamp‚1.7866602299133801e+09 _runtime‚1578.446891615 E8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚41261278 M@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚59049984 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚18379307 KDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚60 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚21635753 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚159469377454080 PIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚77 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 845178880 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 1409842785180 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚159469394558976 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 1412745344512 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 831258624 IDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚19629812 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17104896 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚159939376857064 C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚4289 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚209 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 5368709120 IBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚60 IBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚79 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚61478912 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚21635753 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 5368709120 @7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚5264 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4289 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚21635755 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚19625525 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚19625525 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚140524641247232 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 4552982528 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚41945088 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚140524568176640 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 125829120 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚13468685 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 PCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚59049984 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚4287 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚52426752É­ºGzš¼O VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚160882181848576 G>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4876 IDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚160882122798592 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚262 % _timestamp‚1.7866602299133801e+09 _runtime‚1578.446891615 E8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚41261278 M@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚59049984 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚18379307 KDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚60 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚21635753 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚159469377454080 PIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚77 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 845178880 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 1409842785180 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚159469394558976 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 1412745344512 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 831258624 IDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚19629812 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17104896 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚159939376857064 C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚4289 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚209 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 5368709120 IBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚60 IBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚79 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚61478912 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚21635753 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 5368709120 @7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚5264 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4289 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚21635755 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚19625525 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚19625525 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚140524641247232 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 4552982528 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚41945088 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚140524568176640 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 125829120 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚13468685 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 PCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚59049984 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚4287 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚52426752 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 125829120 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 4654602240 JBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚665 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0ß³µàl2›- _wandbruntime‚1578  _runtime‚1578ˆcß_Ç&œÁM EDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654602240 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚140547928929280 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 1410073357840 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚41272413 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚13470890 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 5368709120 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚21639308 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 BDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚18382246 NGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚62 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚139053896531968 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚159965851610640 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521206 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚158555761213440 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 5494538240 PDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59053056 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚41268118 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚20643840 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 831258624 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚322 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚41948160 NADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚59053056 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 5494538240 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 IBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚60 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 1410032009772 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚160908735258112 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 845178880 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚41937920 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 5368709120 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚4295 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚4293 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚160908676205056 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚21639308 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚41272413 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 1494032397312 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17104896 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚209 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 H@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚665 I@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚4876 G@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚64 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚429 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 125829120 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚322 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚19633105 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 1413018361344 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 1412976413184 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 >5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5264 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚160908735258112 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚13470888 D=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚60 % _timestamp‚1.7866602300924656e+09 D6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚322:¦Ì&ÆM RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 125829120 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚322 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚19633105 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 1413018361344 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 1412976413184 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 >5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5264 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚160908735258112 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚13470888 D=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚60 % _timestamp‚1.7866602300924656e+09 D6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚322 EDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654602240 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚140547928929280 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 1410073357840 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚41272413 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚13470890 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 5368709120 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚21639308 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 BDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚18382246 NGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚62 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚139053896531968 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚159965851610640 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521206 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚158555761213440 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 5494538240 PDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59053056 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚41268118 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚20643840 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 831258624 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚322 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚41948160 NADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚59053056 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 5494538240 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 IBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚60 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 1410032009772 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚160908735258112 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 845178880 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚41937920 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 5368709120 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚4295 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚4293 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚160908676205056 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚21639308 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚41272413 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 1494032397312 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17104896 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚209 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 H@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚665 I@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚4876 G@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚64 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚429à/,K³2ž- _wandbruntime‚1578  _runtime‚1578Ð,È"sŸí trainer/global_step‚5849 _runtime‚1578.624048712 _step‚481 0loss/FRt2/assignment_loss‚0.6843353527760435 /loss/FRt1/detection_loss‚0.5232466942394839 /loss/FRt2/detection_loss‚0.4855441902237218 epoch‚14 % _timestamp‚1.7866602300942242e+09 0loss/FRt1/assignment_loss‚0.7471510461551195 &loss/total_loss‚2.44027728339436847+¡Ox ò trainer/global_step‚5849 _runtime‚1578.624048712 _step‚481 0loss/FRt2/assignment_loss‚0.6843353527760435 /loss/FRt1/detection_loss‚0.5232466942394839 /loss/FRt2/detection_loss‚0.4855441902237218 epoch‚14 % _timestamp‚1.7866602300942242e+09 0loss/FRt1/assignment_loss‚0.7471510461551195 &loss/total_loss‚2.4402772833943684á<?–¡‚(:‹ ‘‹ùÓ¯•—á gpu.0.gpu‚60.0 gpu.0.memory‚11, gpu.0.memoryAllocated‚7.0313262939453125) gpu.0.memoryAllocatedBytes‚ 6039863296 gpu.0.temp‚37.0 gpu.0.powerWatts‚214.374' gpu.0.enforcedPowerLimitWatts‚500.0 gpu.0.powerPercent‚42.8748 gpu.0.smClock‚1410 gpu.0.memoryClock‚1593$ gpu.0.correctedMemoryErrors‚1955# gpu.0.uncorrectedMemoryErrors‚0~ç:s˜¢‚(: ‘‹ùÓµ•›ä# disk.dm-0.out‚53053.10888671875$ memory_percent‚66.77711009979248$ disk./.usageGB‚487.2274398803711( disk.nvme0c0n1.out‚53053.10888671875 disk.sda.in‚15349.44140625! disk.nvme0n1.in‚ 17467.2109375 disk.dm-1.in‚15349.44140625" proc.memory.rssMB‚ 2376.8046875* proc.memory.percent‚14.506864547729492 proc.cpu.threads‚17 network.sent‚ 7.844936e+06 network.recv‚ 3.167994e+06) disk./.usagePercent‚32.69824230142335 disk.sda3.out‚ 31.671875 disk.sda.out‚ 31.671875 disk.drbd2165.out‚0 disk.dm-0.in‚ 17467.2109375 cpu‚27.014062481517666# disk.nvme0c0n1.in‚ 17467.2109375 disk.sda1.in‚0 disk.sda2.in‚0 disk.dm-1.out‚ 31.671875) proc.memory.availableMB‚ 5443.23828125 disk.sda3.in‚15349.44140625 disk.sda1.out‚0 disk.sda2.out‚0& disk.nvme0n1.out‚53053.10888671875 disk.drbd2165.in‚0ÕÅúa2£- _wandbruntime‚1590  _runtime‚1590ò¦¥vƤÀ _step‚482 & lr-AdamW/pg1‚0.0012270483584569765 & lr-AdamW/pg2‚0.0012270483584569765 trainer/global_step‚5899 $ _timestamp‚1.786660242309297e+09 _runtime‚1590.839296831®ŒÒ\˥Š& lr-AdamW/pg1‚0.0012270483584569765 & lr-AdamW/pg2‚0.0012270483584569765 trainer/global_step‚5899 $ _timestamp‚1.786660242309297e+09 _runtime‚1590.839296831 _step‚482âI\Q¾2¦- _wandbruntime‚1590  _runtime‚1590d nQ§·O C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚4289 E8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚41604030 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚19794914 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚18530468 _runtime‚1590.843874778 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚209 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚20643840 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 4654602240 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 1424374055424 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚21813403 JBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚665 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚4287 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚140183197155328 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚159838882136064 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 PCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚58387380 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 125829120 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚18530540 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 trainer/global_step‚5899 M@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚59049984 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚160785511448576 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚161260228344472 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 5494538240 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚140183217799168 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚159838865096704 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚162209885504000 % _timestamp‚1.7866602423101814e+09 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚162209826454016 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚21813405 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚160785494343680 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚162209885504000 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚223 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 5494538240 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 125829120 F>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 122905600 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚322 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚41945088 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 1506011093504 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚322 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚21813403 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚19790627 G>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4876 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚41945088 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 845178880 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 4517085184 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚322 IBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚60 D;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚5264 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚19790627 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚13578935 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 @7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚5264 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17104896 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 1424332110336 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚41940992 BDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚162209826454016 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 1424374055424 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚429 _step‚483 KBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚4876 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 1506053034496 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 4552982528 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚160785494343680 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚13578933 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚21813405 PCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚59049984 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚41608319 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚19794914 IBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚72 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 IDeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4289 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚4951535 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚262øQ­6Â'¨¼O M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚41608319 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚19794914 IBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚72 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 IDeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4289 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚4951535 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚262 C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚4289 E8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚41604030 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚19794914 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚18530468 _runtime‚1590.843874778 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚209 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚20643840 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 4654602240 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 1424374055424 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚21813403 JBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚665 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚4287 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚140183197155328 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚159838882136064 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 PCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚58387380 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 125829120 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚18530540 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 trainer/global_step‚5899 M@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚59049984 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚160785511448576 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚161260228344472 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 5494538240 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚140183217799168 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚159838865096704 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚162209885504000 % _timestamp‚1.7866602423101814e+09 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚162209826454016 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚21813405 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚160785494343680 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚162209885504000 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚223 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 5494538240 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 125829120 F>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 122905600 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚322 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚41945088 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 1506011093504 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚322 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚21813403 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚19790627 G>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4876 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚41945088 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 845178880 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 4517085184 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚322 IBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚60 D;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚5264 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚19790627 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚13578935 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 @7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚5264 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17104896 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 1424332110336 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚41940992 BDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚162209826454016 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 1424374055424 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚429 _step‚483 KBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚4876 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 1506053034496 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 4552982528 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚160785494343680 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚13578933 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚21813405 PCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚59049984 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0ã|û¼2©- _wandbruntime‚1591  _runtime‚1591È™CÇ&ªÁM NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚21816956 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 5494538240 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚262 D=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚60 I@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚4876 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚44035072 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 122905600 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚161286703098048 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚161286644710620 trainer/global_step‚5899 K>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59053056 >5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5264 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚4952429 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚141712566183424 SFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚64678912 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚41610870 NADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚59053056 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚209 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚18533569 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 1421593820380 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 5368709120 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚4952366 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 1421635168448 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 122905600 PDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654602240 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚429 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 G@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚60 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 5368709120 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚41948160 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚223 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚262 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚4293 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚162236379860480 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚18533504 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚159865050890240 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚322 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 4654602240 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚160811833786368 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17104896 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚141712501504512 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚21816958 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚162236438913536 H@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚665 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 845178880 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 125829120 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚160811833786368 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚41610870 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚61478912 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚4293 NGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚63 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚140206265171968 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 122306526 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚21816956 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521206 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 4517085184 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 5494538240 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 125829120 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚4295 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚19793914 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 1506256976384 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 1424563179008 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚13581140 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 125829120 _runtime‚1591.002047998 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚322 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚41948160 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 1424605127168 EDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚41615165 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚159865067929600 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 4552982528 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 1506301011456 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 1424563179008 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚19793914 G@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚65 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚322 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚19798207 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚262 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 IBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚60 DDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59053056 >5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5264 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚4952429 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚141712566183424 SFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚64678912 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚41610870 NADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚59053056 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚209 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚18533569 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 1421593820380 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 5368709120 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚4952366 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 1421635168448 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 122905600 PDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654602240 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚429 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 G@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚60 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 5368709120 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚41948160 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚223 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚262 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚4293 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚162236379860480 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚18533504 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚159865050890240 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚322 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 4654602240 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚160811833786368 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17104896 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚141712501504512 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚21816958 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚162236438913536 H@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚665 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 845178880 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 125829120 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚160811833786368 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚41610870 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚61478912 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚4293 NGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚63 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚140206265171968 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 122306526 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚21816956 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521206 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 4517085184 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 5494538240 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 125829120 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚4295 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚19793914 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 1506256976384 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 1424563179008 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚13581140 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 125829120 _runtime‚1591.002047998 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚322 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚41948160 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 1424605127168 EDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚41615165 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚159865067929600 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 4552982528 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 1506301011456 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 1424563179008 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚19793914 G@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚65 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚322 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚19798207 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚262 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 IBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚60 DDeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4289 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚48232448 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚322 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚322 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 5494538240 C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚4289 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚163537589159424 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚162101611233280 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 125829120 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚21991053 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚21991055 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚4992256 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 831258624 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚322 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚21991055 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚163537530109440 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚161148154773504 F>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 G>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4876 M@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚59049984 UHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚68876288 % _timestamp‚1.7866602549240735e+09 @7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚5264 IBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚61 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚13689183 PIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚59 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 125829120 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 122905600 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚21991053 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚209 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 1518326003200 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚161148171812864 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚262 IBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚60 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 1518277770752 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 5494538240 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚41945088 JBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚665 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 1435960821248 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚142853843566080 BDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚162581138219260 PCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚58387380 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚322 IDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚163537530109440 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚161148154773504 F>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 G>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4876 M@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚59049984 UHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚68876288 % _timestamp‚1.7866602549240735e+09 @7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚5264 IBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚61 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚13689183 PIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚59 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 125829120 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 122905600 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚21991053 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚209 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 1518326003200 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚161148171812864 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚262 IBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚60 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 1518277770752 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 5494538240 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚41945088 JBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚665 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 1435960821248 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚142853843566080 BDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚162581138219260 PCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚58387380 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚322 IDeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4289 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚48232448 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚322 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚322 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 5494538240 C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚4289 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚163537589159424 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚162101611233280 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 125829120çˆÉ 2µ- _wandbruntime‚1603  _runtime‚1603L†“éÇ&¶ÁM NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚19963309 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17104896 G@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚56 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 5368709120 NADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚59053056 NADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚58387428 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 125829120 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 122905600 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚262 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚162607554585456 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 5494538240 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 125829120 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚21994608 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚4293 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 122905600 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚61478912 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 5494538240 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚41953622 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚19963309 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚13691388 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 D=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚60 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚21994608 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚20643840 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚161174340567040 D8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 G@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚60 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚163564142568960 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521206 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 1433196979056 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 125829120 _step‚488 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚4295 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚223 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚161174357606400 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚163564142568960 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 1436191892992 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚41957917 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚19959016 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 1518525259264 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚41957917 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚162127950675968 _runtime‚1603.618825425 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 4654602240 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 1436191892992 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚41948160 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚141358633811968 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚322 EDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 IBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚60 trainer/global_step‚5949 K>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59053056 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 4552982528 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 5494538240 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654602240 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚4293 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 831258624 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚162607496198028 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚142877138427392 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚322 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚41348068 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚322 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚162127933571072 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚18684359 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 122306526 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚41953622 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 BDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 5368709120 >5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5264 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚162127933571072 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚322­ÞEl·ÆM UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 1436149944832 B9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚5264 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚21994606 H@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚665 I@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚4876 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚44035072 D>DeviceStatsMonitor.on_trainVΰË$_batch_end/reserved_bytes.all.freed‚0 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 5368709120 >5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5264 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚162127933571072 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚322 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚19963309 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17104896 G@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚56 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 5368709120 NADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚59053056 NADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚58387428 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 125829120 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 122905600 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚262 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚162607554585456 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 5494538240 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 125829120 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚21994608 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚4293 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 122905600 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚61478912 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 5494538240 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚41953622 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚19963309 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚13691388 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 D=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚60 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚21994608 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚20643840 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚161174340567040 D8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 G@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚60 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚163564142568960 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521206 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 1433196979056 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 125829120 _step‚488 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚4295 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚223 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚161174357606400 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚163564142568960 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 1436191892992 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚41957917 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚19959016 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 1518525259264 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚41957917 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚162127950675968 _runtime‚1603.618825425 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 4654602240 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 1436191892992 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚41948160 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚141358633811968 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚322 EDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 IBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚60 trainer/global_step‚5949 K>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59053056 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 4552982528 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 5494538240 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654602240 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚4293 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 831258624 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚162607496198028 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚142877138427392 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚322 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚41348068 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚322 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚162127933571072 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚18684359 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 122306526 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚41953622 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 BDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚13799435 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚142487955079168 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 4517085184 PCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚59049984 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚142487934435328 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 1530547544576 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚5033234 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚144018544564736 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚162457461489664 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚262 % _timestamp‚1.7866602673957133e+09 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 1447547587072 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚5033175 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚322 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 IBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚60 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚163417728122880 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚20125118 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚163901931319288 @7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚5264 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚163417728122880 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 5368709120 C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚4289 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 1447505641984 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 845178880 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17104896 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 1444528217004 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 5494538240 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚164865233764864 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚429 KBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚4876 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚163901989706668 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚262 IBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚61 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚164865292814848 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚41945088 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 1447505641984 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚20643840 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 125829120 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚20120831 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚164865233764864 D;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚5264 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚144018481979904 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚61478912 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚322 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚41945088 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚42293823 PIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚59 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 831258624 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚162457444450304 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 5494538240 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚322 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 4552982528 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 122905600 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 4628521206 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 _step‚491 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4289 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚22168705 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 IDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4876 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚209 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 trainer/global_step‚5999 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚223 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0°!~Â'¼O E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚322 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 F?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚60 F>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚13799435 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚142487955079168 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 4517085184 PCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚59049984 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚142487934435328 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 1530547544576 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚5033234 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚144018544564736 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚162457461489664 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚262 % _timestamp‚1.7866602673957133e+09 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 1447547587072 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚5033175 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚322 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 IBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚60 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚163417728122880 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚20125118 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚163901931319288 @7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚5264 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚163417728122880 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 5368709120 C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚4289 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 1447505641984 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 845178880 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17104896 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 1444528217004 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 5494538240 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚164865233764864 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚429 KBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚4876 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚163901989706668 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚262 IBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚61 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚164865292814848 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚41945088 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 1447505641984 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚20643840 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 125829120 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚20120831 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚164865233764864 D;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚5264 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚144018481979904 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚61478912 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚322 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚41945088 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚42293823 PIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚59 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 831258624 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚162457444450304 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 5494538240 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚322 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 4552982528 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 122905600 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 4628521206 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 _step‚491 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4289 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚22168705 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 IDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4876 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚209 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 trainer/global_step‚5999 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚223 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚4287 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17104896 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚13799433 BDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521206 _step‚492 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚13801640 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚164891846224384 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 5368709120 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚22172256 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚164891846224384 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 1447778658816 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚41948160 EDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚13801638 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚223 trainer/global_step‚5999 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 4654602240 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17104896 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚18835677 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 5368709120 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚262 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚20128411 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚142511002451968 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 5494538240 IBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚60 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚42300669 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 4552982528 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚42300669 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 1444717441596 DDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚18835611 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 125829120 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚42296374 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚61478912 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 125829120 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚20643840 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚163928347685436 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 5494538240 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚20128411 G@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚66 K>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59053056 H@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚665 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 1530788708864 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚164891787171328 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚4293 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17104896 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚5033973 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚46132224 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 5494538240 G@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚60 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚163444067565568 SFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚66776064 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654602240 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚163928406072864 >5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5264 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 4552982528 NADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚58387428 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚322 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 5368709120 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚322 NADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚59053056 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚5034037 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚322 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 4517085184 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 1444758789664 _runtime‚1616.091589351 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚20124118 B9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚5264 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 % _timestamp‚1.7866602675612686e+09 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 122905600 NGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚64 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚41948160 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚163444050460672 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚209 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 B6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚322 D=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚60 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚22172256k+¬\Ì&ÅÆM NADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚59053056 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚5034037 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚322 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 4517085184 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 1444758789664 _runtime‚1616.091589351 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚20124118 B9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚5264 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 % _timestamp‚1.7866602675612686e+09 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 122905600 NGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚64 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚41948160 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚163444050460672 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚209 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 B6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚322 D=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚60 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚22172256 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚42296374 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 845178880 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚22172258 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚163444067565568 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚41348068 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚4295 EDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521206 _step‚492 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚13801640 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚164891846224384 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 5368709120 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚22172256 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚164891846224384 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 1447778658816 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚41948160 EDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚13801638 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚223 trainer/global_step‚5999 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 4654602240 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17104896 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚18835677 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 5368709120 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚262 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚20128411 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚142511002451968 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 5494538240 IBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚60 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚42300669 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 4552982528 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚42300669 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 1444717441596 DDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚18835611 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 125829120 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚42296374 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚61478912 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 125829120 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚20643840 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚163928347685436 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 5494538240 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚20128411 G@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚66 K>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59053056 H@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚665 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 1530788708864 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚164891787171328 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚4293 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17104896 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚5033973 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚46132224 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 5494538240 G@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚60 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚163444067565568 SFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚66776064 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654602240 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚163928406072864 >5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5264 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 4552982528 NADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚58387428 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚322 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 5368709120 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚322ìB1fà2Æ- _wandbruntime‚1616  _runtime‚1616÷ 2-qÇë 0loss/FRt1/assignment_loss‚0.9046782782859037 0loss/FRt2/assignment_loss‚0.7892642166990582 .loss/FRt1/detection_loss‚0.603417660247655 0loss/FRt2/detection_loss‚0.47900786933445194 epoch‚14 _step‚493 %loss/total_loss‚2.776368024567069 trainer/global_step‚5999 $ _timestamp‚1.786660267562337e+09 _runtime‚1616.092149088Ê«Ï¿vÈð 0loss/FRt1/assignment_loss‚0.9046782782859037 0loss/FRt2/assignment_loss‚0.7892642166990582 .loss/FRt1/detection_loss‚0.603417660247655 0loss/FRt2/detection_loss‚0.47900786933445194 epoch‚14 _step‚493 %loss/total_loss‚2.776368024567069 trainer/global_step‚5999 $ _timestamp‚1.786660267562337e+09 _runtime‚1616.092149088í1¾”É‚(:‰ ¯‹ùÓÞË‚á gpu.0.gpu‚54.0 gpu.0.memory‚6, gpu.0.memoryAllocated‚7.0313262939453125) gpu.0.memoryAllocatedBytes‚ 6039863296 gpu.0.temp‚35.0 gpu.0.powerWatts‚119.555' gpu.0.enforcedPowerLimitWatts‚500.0 gpu.0.powerPercent‚23.911 gpu.0.smClock‚1410 gpu.0.memoryClock‚1593$ gpu.0.correctedMemoryErrors‚1955# gpu.0.uncorrectedMemoryErrors‚0ɇ1CšÊ‚(: ¯‹ùÓÄ®¦ã disk.drbd2165.out‚0 disk.sda3.out‚ 32.296875$ disk.nvme0c0n1.in‚23359.56640625 disk.sda.in‚15349.44140625 disk.sda1.out‚0$ memory_percent‚66.77699089050293 network.sent‚ 8.014893e+06" disk.nvme0n1.in‚23359.56640625 disk.dm-1.out‚ 32.296875 disk.sda1.in‚0" proc.memory.rssMB‚ 2376.8046875 proc.cpu.threads‚17& disk.nvme0n1.out‚53101.84326171875 disk.drbd2165.in‚0# disk.dm-0.out‚53101.84326171875 disk.sda3.in‚15349.44140625 disk.sda.out‚ 32.296875 disk.sda2.in‚0 disk.sda2.out‚0( proc.memory.availableMB‚ 5443.2578125 network.recv‚ 3.232899e+06% disk./.usageGB‚487.29007720947266 disk.dm-0.in‚23359.56640625( disk.nvme0c0n1.out‚53101.84326171875* proc.memory.percent‚14.506864547729492 cpu‚27.03073628118445) disk./.usagePercent‚32.70244594513558 disk.dm-1.in‚15349.44140625Ð'9±2Ë- _wandbruntime‚1621  _runtime‚1621*´“9ÞÌØ *loss/val_total_loss‚2.6025434070754945 2Purity/*FRt/event_proportion‚0.669805645942688 3Purity/1FRt/event_proportion‚0.6297432780265808 *validation_accuracy‚0.1887696534395218 Purity/0FRt/event_purity‚1 2Purity/2FRt/event_proportion‚0.040062315762043 +jet/accuracy_1_of_1‚0.29342591762542725 .particle/accuracy_2_of_2‚0.628311812877655 trainer/global_step‚5999 % _timestamp‚1.7866602727428603e+09 0Purity/*FRt/event_purity‚0.43665844202041626 .Purity/1FRt/FRt_purity‚0.44001060724258423 +particle/specificity‚0.6481533646583557 _runtime‚1621.272931521 _step‚494 -Purity/*FRt/FRt_purity‚0.4495137929916382 0Purity/1FRt/event_purity‚0.44001060724258423 -Purity/2FRt/FRt_purity‚0.5232205390930176 particle/accuracy_1_of_1‚1 +particle/sensitivity‚0.6132726073265076 'particle/f_score‚0.5444328188896179 3Purity/0FRt/event_proportion‚0.3301943838596344 (particle/accuracy‚0.6357538104057312 0Purity/2FRt/event_purity‚0.38185518980026245 *jet/accuracy_1_of_2‚0.5447115898132324 /particle/accuracy_1_of_2‚0.8878605365753174 epoch‚14 *jet/accuracy_2_of_2‚0.1887696534395218 6validation_average_jet_accuracy‚0.2978443503379822Ôp^ãÍÝ 2Purity/2FRt/event_proportion‚0.040062315762043 +jet/accuracy_1_of_1‚0.29342591762542725 .particle/accuracy_2_of_2‚0.628311812877655 trainer/global_step‚5999 % _timestamp‚1.7866602727428603e+09 0Purity/*FRt/event_purity‚0.43665844202041626 .Purity/1FRt/FRt_purity‚0.44001060724258423 +particle/specificity‚0.6481533646583557 _runtime‚1621.272931521 _step‚494 -Purity/*FRt/FRt_purity‚0.4495137929916382 0Purity/1FRt/event_purity‚0.44001060724258423 -Purity/2FRt/FRt_purity‚0.5232205390930176 particle/accuracy_1_of_1‚1 +particle/sensitivity‚0.6132726073265076 'particle/f_score‚0.5444328188896179 3Purity/0FRt/event_proportion‚0.3301943838596344 (particle/accuracy‚0.6357538104057312 0Purity/2FRt/event_purity‚0.38185518980026245 *jet/accuracy_1_of_2‚0.5447115898132324 /particle/accuracy_1_of_2‚0.8878605365753174 epoch‚14 *jet/accuracy_2_of_2‚0.1887696534395218 6validation_average_jet_accuracy‚0.2978443503379822 *loss/val_total_loss‚2.6025434070754945 2Purity/*FRt/event_proportion‚0.669805645942688 3Purity/1FRt/event_proportion‚0.6297432780265808 *validation_accuracy‚0.1887696534395218 Purity/0FRt/event_purity‚1î(³³2Î- _wandbruntime‚1634  _runtime‚1634ßÿxÇÏÁ _step‚495 & lr-AdamW/pg1‚0.0012129976087423625 & lr-AdamW/pg2‚0.0012129976087423625 trainer/global_step‚6049 % _timestamp‚1.7866602855158434e+09 _runtime‚1634.047901511hQƒÌÐÆ & lr-AdamW/pg1‚0.0012129976087423625 & lr-AdamW/pg2‚0.0012129976087423625 trainer/global_step‚6049 % _timestamp‚1.7866602855158434e+09 _runtime‚1634.047901511 _step‚495ï›X2Ñ- _wandbruntime‚1634  _runtime‚1634ó+µ2´Ò·O \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚50329600 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 trainer/global_step‚6049 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚209 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 1463160131584 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚41945088 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚13950299 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 5494538240 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚166587942359040 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚41348020 C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚4289 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚4287 G>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4876 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 1463160131584 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚164147633618944 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17104896 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4289 IBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚60 BDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚262 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 4628521206 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 125829120 KDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚60 % _timestamp‚1.7866602855165243e+09 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚19032638 IBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚72 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚429 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚145486491362304 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 IDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 JBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚665 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚143939573383168 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 5494538240 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚322 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚223 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚22406961 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚19032566 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚5082269 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚143939594027008 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚164147650658304 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚20308120 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚20643840 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚165607748270637 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 5368709120 @7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚5264 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚165607806658017 _runtime‚1634.048009755 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 4654602240 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚166588001409024 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 1463202076672 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚5082339 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 PCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚58387380 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚165124782227456 PIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚70 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 125829120 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 JBDeviceStatsMonitor.on_train_batch_st–ÜÉ art/segment.large_pool.current‚262 _step‚496 E8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚42715079 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 1463202076672 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚165124799332352 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚13950297 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 122306526 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 122905600 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚61478912 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 5368709120 F?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚60 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 UHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚70973440 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 1460155999713 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 125829120 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚22406959 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 4654602240 IDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚166587942359040 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚41348020 C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚4289 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚4287 G>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4876 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 1463160131584 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚164147633618944 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17104896 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4289 IBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚60 BDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚262 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 4628521206 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 125829120 KDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚60 % _timestamp‚1.7866602855165243e+09 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚19032638 IBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚72 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚429 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚145486491362304 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 IDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 JBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚665 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚143939573383168 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 5494538240 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚322 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚223 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚22406961ðÁ*O2Ô- _wandbruntime‚1634  _runtime‚1634ñ¢cÇ&ÕÁM YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 1463433148416 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚22410514 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521206 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 1460345224305 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 5494538240 _step‚497 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚165151121670144 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚143962620755968 % _timestamp‚1.7866602856909232e+09 K>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59053056 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚5083115 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚61478912 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 IBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚60 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 5368709120 H@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚665 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚19035619 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚42726214 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚20311407 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚165634164636785 D=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚60 I@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚4876 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚166614554818560 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚5083051 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚262 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚4295 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 4654602240 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 4552982528 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 845178880 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 125829120 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 1463391200256 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚223 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚145509784814592 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚322 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 1463433148416 B9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚5264 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚19035553 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚262 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚166614554818560 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 5368709120 trainer/global_step‚6049 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 G@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚66 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚429 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 4552982528 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654602240 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚4293 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚20643840 B6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚322 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 122306526 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 E6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚322 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 _runtime‚1634.221321116 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚42721919 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚165151104565248 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚13952502 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚262 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚13952504 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚48229376 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚41348068 G@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚60 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚22410512 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚20311407 NGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚64 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 1547164058624 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 >5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5264 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚165151104565248 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚42721919 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 122905600 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚20315700 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚42726214 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚20315700 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚165634223024213 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 125829120 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 1463391200256 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 4517085184ŸÎí3Ì&ÖÆM K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚42726214 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚20315700 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚165634223024213 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 125829120 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 1463391200256 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 4517085184 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 1463433148416 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚22410514 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521206 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 1460345224305 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 5494538240 _step‚497 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚165151121670144 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚143962620755968 % _timestamp‚1.7866602856909232e+09 K>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59053056 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚5083115 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚61478912 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 IBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚60 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 5368709120 H@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚665 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚19035619 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚42726214 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚20311407 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚165634164636785 D=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚60 I@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚4876 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚166614554818560 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚5083051 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚262 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚4295 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 4654602240 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 4552982528 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 845178880 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 125829120 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 1463391200256 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚223 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚145509784814592 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚322 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 1463433148416 B9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚5264 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚19035553 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚262 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚166614554818560 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 5368709120 trainer/global_step‚6049 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 G@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚66 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚429 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 4552982528 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654602240 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚4293 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚20643840 B6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚322 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 122306526 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 E6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚322 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 _runtime‚1634.221321116 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚42721919 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚165151104565248 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚13952502 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚262 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚13952504 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚48229376 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚41348068 G@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚60 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚22410512 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚20311407 NGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚64 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 1547164058624 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 >5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5264 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚165151104565248 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚42721919 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 122905600 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚20315700ñs¶³2×- _wandbruntime‚1634  _runtime‚1634 þDÇ Øí %loss/total_loss‚2.373565059191254 trainer/global_step‚6049 % _timestamp‚1.7866602856920822e+09 _step‚498 0loss/FRt1/assignment_loss‚0.7397162922441787 0loss/FRt2/assignment_loss‚0.5635732299617372 /loss/FRt1/detection_loss‚0.5766186<À‘×f275265047 0loss/FRt2/detection_loss‚0.49365690945883334 epoch‚15 _runtime‚1634.221954884ì†ÖxÙò 0loss/FRt1/assignment_loss‚0.7397162922441787 0loss/FRt2/assignment_loss‚0.5635732299617372 /loss/FRt1/detection_loss‚0.5766186275265047 0loss/FRt2/detection_loss‚0.49365690945883334 epoch‚15 _runtime‚1634.221954884 %loss/total_loss‚2.373565059191254 trainer/global_step‚6049 % _timestamp‚1.7866602856920822e+09 _step‚498òŽ× è¡Ú‚(:– ¾‹ùÓ’ʯá gpu.0.gpu‚60.0 gpu.0.memory‚11, gpu.0.memoryAllocated‚7.0313262939453125) gpu.0.memoryAllocatedBytes‚ 6039863296 gpu.0.temp‚34.0 gpu.0.powerWatts‚132.573' gpu.0.enforcedPowerLimitWatts‚500.0) gpu.0.powerPercent‚26.514599999999998 gpu.0.smClock‚1410 gpu.0.memoryClock‚1593$ gpu.0.correctedMemoryErrors‚1955# gpu.0.uncorrectedMemoryErrors‚0´„\®ŒÛ‚(: ¾‹ùÓÏØ„ä network.recv‚ 3.265559e+06& disk.nvme0c0n1.out‚53148.220703125 disk.sda1.in‚0 disk.sda1.out‚0 disk.nvme0n1.in‚ 23362.875$ disk.nvme0n1.out‚53148.220703125 disk.drbd2165.out‚0" proc.memory.rssMB‚ 2376.8046875 disk.sda3.in‚15349.44140625 disk.nvme0c0n1.in‚ 23362.875 disk.sda.in‚15349.44140625 disk.dm-1.out‚ 32.58984375 disk.drbd2165.in‚0$ memory_percent‚66.78011417388916 proc.cpu.threads‚17 disk.sda2.in‚0 disk.dm-1.in‚15349.44140625 disk.dm-0.in‚ 23362.875! disk.dm-0.out‚53148.220703125 cpu‚27.036565905393037 network.sent‚ 8.089856e+06( disk./.usagePercent‚32.7018607119123% disk./.usageGB‚487.28135681152344 disk.sda3.out‚ 32.58984375 disk.sda.out‚ 32.58984375 disk.sda2.out‚0) proc.memory.availableMB‚ 5442.74609375* proc.memory.percent‚14.506864547729492&««ì2Ü- _wandbruntime‚1646  _runtime‚1646Ti.ŒÇÝÁ & lr-AdamW/pg1‚0.0012082541475332879 & lr-AdamW/pg2‚0.0012082541475332879 trainer/global_step‚6099 % _timestamp‚1.7866602980011263e+09 _runtime‚1646.531154636 _step‚499¨éܨÌÞÆ _runtime‚1646.531154636 _step‚499 & lr-AdamW/pg1‚0.0012082541475332879 & lr-AdamW/pg2‚0.0012082541475332879 trainer/global_step‚6099 % _timestamp‚1.7866602980011263e+09óRÜ2¦2ß- _wandbruntime‚1646  _runtime‚1646”ñ8Û½'à·O M@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚59049984 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚22584609 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚4287 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚20473222 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 122306526 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚22584609 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 5368709120 IBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚60 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚167915705064448 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚209 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 125829120 C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚4289 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚166440916221952 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚145091962667008 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 1559255153152 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 4628521206 _step‚500 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 4654602240 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚41348020 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚262 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 1474788842496 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚14060547 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚5122939 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚166928599758045 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 5494538240 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚4287 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚322 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚322 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚41945088 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 125829120 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚167915705064448 KBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚4876 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 4517085184 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 1471717810321 % _timestamp‚1.7866602980020514e+09 _runtime‚1646.535892246 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17104896 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚5123003 UHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚66779136 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚146651151041024 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚262 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚167915646014464 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚22584611 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚145091942023168 D;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚5264 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚61478912 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 trainer/global_step‚6099 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 4552982528 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚166440899117056 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 122905600 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 845178880 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 5494538240 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 125829120 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚20477509 JBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚665 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚19183552 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚46135296 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚165456923295744 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚20473222 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 122905600 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚322 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚22584611 F>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 1474746897408 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 5368709120 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚14060549 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 G>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4876 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚165456940335104 IDeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4289 PIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚64 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 5494538240 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚322 @7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚5264 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚167915646014464 BDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 1474746897408 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 5368709120 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚14060549 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 G>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4876 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚165456940335104 IDeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4289 PIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚64 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 5494538240 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚322 @7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚5264 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚167915646014464 BDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚41348020 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚262 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 1474788842496 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚14060547 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚5122939 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚166928599758045 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 5494538240 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚4287 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚322 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚322 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚41945088 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 125829120 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚167915705064448 KBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚4876 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 4517085184 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 1471717810321 % _timestamp‚1.7866602980020514e+09 _runtime‚1646.535892246 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17104896 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚5123003 UHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚66779136 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚146651151041024 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚262 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚167915646014464 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚22584611 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚145091942023168 D;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚5264 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚61478912 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 trainer/global_step‚6099ôJϪ2â- _wandbruntime‚1646  _runtime‚1646›U3ßÇ&ãÁM I@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚4876 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚5123813 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 125829120 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚43064671 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521206 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚322 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚20476509 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚146674442199552 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 125829120 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 1559496838656 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚165483126128640 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚262 trainer/global_step‚6099 EDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚19186496 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 4517085184 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚43068966 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚20480802 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚20476509 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚429 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 5494538240 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 5368709120 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚262 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚4293 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚145114989395968 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚145115010039808 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚61478912 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚4293 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚5123744 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 1471907034913 _runtime‚1646.700954589 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚14062754 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚209 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚322 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚166467238559744 G@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚71 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 4654602240 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚166467238559744 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17104896 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚41348068 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚322 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 845178880 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚20643840 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚166955074511621 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚43064671 DDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654602240 NADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚58387428 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚262 G@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚60 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚166467221454848 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚43068966 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚223 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚322 B5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5264 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚19186567 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 B9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚5264 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚14062752 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 5494538240 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 5368709120 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 125829120 EDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59053056ÝÁÉ+äÆM C=DeviceStatsMonitor.on_train_batcha´n¡&_end/oversize_segments.freed‚0 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 1471907034913 _runtime‚1646.700954589 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚14062754 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚209 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚322 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚166467238559744 G@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚71 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 4654602240 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚166467238559744 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17104896 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚41348068 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚322 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 845178880 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚20643840 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚166955074511621 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚43064671 DDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654602240 NADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚58387428 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚262 G@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚60 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚166467221454848 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚43068966 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚223 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚322 B5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5264 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚19186567 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 B9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚5264 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚14062752 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 5494538240 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 5368709120 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 125829120 EDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59053056 I@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚4876 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚5123813 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 125829120 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚43064671 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521206 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚322 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚20476509 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚146674442199552 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 125829120 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 1559496838656 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚165483126128640 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚262 trainer/global_step‚6099 EDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚19186496 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 4517085184 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚43068966 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚20480802 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚20476509 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚429 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 5494538240 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 5368709120 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚262 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚4293 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚145114989395968 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚145115010039808 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚61478912 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚4293 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚5123744 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1õí‹¥°2å- _wandbruntime‚1646  _runtime‚1646ÓÁtæî 0loss/FRt2/detection_loss‚0.47780297422079726 &loss/total_loss‚2.3715424076949985 epoch‚15 trainer/global_step‚6099 0loss/FRt2/assignment_loss‚0.6038851757162754 % _timestamp‚1.7866602981716363e+09 _runtime‚1646.701484751 _step‚502 0loss/FRt1/assignment_loss‚0.7762761121033698 /loss/FRt1/detection_loss‚0.5135781456545557yu÷?yçó 0loss/FRt1/assignment_loss‚0.7762761121033698 /loss/FRt1/detection_loss‚0.5135781456545557 0loss/FRt2/detection_loss‚0.47780297422079726 &loss/total_loss‚2.3715424076949985 epoch‚15 trainer/global_step‚6099 0loss/FRt2/assignment_loss‚0.6038851757162754 % _timestamp‚1.7866602981716363e+09 _runtime‚1646.701484751 _step‚502öÎS è‚(:• Í‹ùÓ½è¤á gpu.0.gpu‚28.0 gpu.0.memory‚4, gpu.0.memoryAllocated‚7.0313262939453125) gpu.0.memoryAllocatedBytes‚ 6039863296 gpu.0.temp‚35.0 gpu.0.powerWatts‚153.623' gpu.0.enforcedPowerLimitWatts‚500.0) gpu.0.powerPercent‚30.724599999999995 gpu.0.smClock‚1410 gpu.0.memoryClock‚1593$ gpu.0.correctedMemoryErrors‚1955# gpu.0.uncorrectedMemoryErrors‚0|ª“é‚(:ˆ Í‹ùÓ·öâ$ disk./.usageGB‚487.3470001220703 disk.sda3.in‚15349.44140625 disk.sda.in‚15349.44140625 disk.sda.out‚ 32.74609375 disk.nvme0n1.in‚ 23441.0625 disk.dm-1.in‚15349.44140625 disk.dm-1.out‚ 32.74609375 disk.dm-0.in‚ 23441.0625 network.recv‚ 3.291039e+06 disk.sda3.out‚ 32.74609375 disk.nvme0c0n1.in‚ 23441.0625 disk.sda1.out‚0 disk.drbd2165.in‚0# disk.dm-0.out‚53158.43994140625* proc.memory.percent‚14.506864547729492 network.sent‚ 8.159333e+06 disk.sda1.in‚0 disk.drbd2165.out‚0$ memory_percent‚66.77660942077637( proc.memory.availableMB‚ 5443.3203125 cpu‚27.03701824361914) disk./.usagePercent‚32.70626608956152( disk.nvme0c0n1.out‚53158.43994140625 disk.sda2.in‚0 disk.sda2.out‚0& disk.nvme0n1.out‚53158.43994140625" proc.memory.rssMB‚ 2376.8046875 proc.cpu.threads‚17gøÉa2ê- _wandbruntime‚1659  _runtime‚1659 OGVÃë½ # _timestamp‚1.78666031070703e+09 _runtime‚1659.237063064 _step‚503 % lr-AdamW/pg1‚0.001203481253516635 % lr-AdamW/pg2‚0.001203481253516635 trainer/global_step‚6149÷ˆuÈì _step‚503 % lr-AdamW/pg1‚0.001203481253516635 % lr-AdamW/pg2‚0.001203481253516635 trainer/global_step‚6149 # _timestamp‚1.78666031070703e+09 _runtime‚1659.237063064÷œ9£{2í- _wandbruntime‚1659  _runtime‚1659v×m½'î·O C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚4289 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚322 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚166766230011904 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 IBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚60 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 4654602240 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4289 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚5164256 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚262 @7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚5264 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚41945088 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚22762261 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 G>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4876 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚169243349669888 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17104896 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚169243349669888 IBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚71 UHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚62584832 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 845178880 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚262 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 4654602240 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚167757033111552 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚167757016006656 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚22762259 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚20642611 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚20638324 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚14170797 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚209 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚20642611 PIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚69 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 4628521206 KDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚60 % _timestamp‚1.7866603107079744e+09 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17104896 KBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚4876 BDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 1486333663232 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚19335055 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚61478912 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 _step‚504 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚22762259 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚147815867153408 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚168249451245453 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚166766212972544 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 5368709120 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 1486375608320 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 JBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚665 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 122306526 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 125829120 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 125829120 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚41945088 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚19334984 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 1483279620929 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 5368709120 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 122905600 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚4287 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚147815804568576 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 122905600 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚4287 D;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚5264 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 125829120 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚322 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 4552982528 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚20643840 _runtime‚1659.238244616 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚168249509632833 M@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚59049984 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚22762261 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 1571535846400 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚169243408719872 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚169243408719872 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 trainer/global_step‚6149 F>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚20638324 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚5164187 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 PCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚58387380 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 5368709120ñi^U 'ï¼O M@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚59049984 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚22762261 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 1571535846400 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚169243408719872 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚169243408719872 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 trainer/global_step‚6149 F>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚20638324 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚5164187 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 PCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚58387380 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 5368709120 C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚4289 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚322 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚166766230011904 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 IBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚60 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 4654602240 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4289 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚5164256 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚262 @7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚5264 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚41945088 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚22762261 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 G>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4876 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚169243349669888 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17104896 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚169243349669888 IBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚71 UHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚62584832 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 845178880 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚262 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 4654602240 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚167757033111552 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚167757016006656 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚22762259 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚20642611 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚20638324 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚14170797 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚209 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚20642611 PIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚69 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 4628521206 KDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚60 % _timestamp‚1.7866603107079744e+09 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17104896 KBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚4876 BDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 1486333663232 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚19335055 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚61478912 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 _step‚504 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚22762259 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚147815867153408 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚168249451245453 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚166766212972544 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 5368709120 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 1486375608320 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 JBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚665 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 122306526 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 125829120 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 125829120 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚41945088 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚19334984 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 1483279620929 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 5368709120 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 122905600 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚4287 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚147815804568576 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 122905600 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚4287 D;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚5264 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 125829120 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚322 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 4552982528 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚20643840 _runtime‚1659.23824XtQêµ4616 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚168249509632833øcP2ð- _wandbruntime‚1659  _runtime‚1659Èç?YÆ&ñÀM HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521206 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚20641611 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚147839161978880 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚146267378679808 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚262 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚169269903076352 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 4552982528 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 5368709120 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 4552982528 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚429 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 125829120 IBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚60 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚43407423 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚169269962129408 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚262 _step‚505 I@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚4876 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚20645904 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚43407423 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚168275925999029 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚41348068 BDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654602240 G@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚66 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚322 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚166792398766080 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚262 _runtime‚1659.400175422 DDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59053056 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 4654602240 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚41948160 H@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚665 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 $ _timestamp‚1.786660310869891e+09 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚43411718 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚19338065 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 4517085184 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚4295 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17104896 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚22765812 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 1571739264000 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚4293 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚4293 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚14173004 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 1571783299072 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚61478912 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 122905600 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 1483468845521 trainer/global_step‚6149 >5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5264 B9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚5264 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚41948160 SFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚64678912 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚322 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚168275867611601 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 5368709120 NADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚59053056 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚14173002 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 845178880 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚322 NADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚58387428 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 125829120 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚5164997 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚20643840 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 831258624 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚166792415805440 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 5494538240 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 5368709120 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 122905600 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚167783355449344 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 1486564731904 D=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚60 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17104896 NGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚64 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚44035072 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 5494538240 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 G@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚60 EDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚19337999ªe'ÕË&òÅM _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚146267358035968 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 PDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚19337999 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521206 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚20641611 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚147839161978880 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚146267378679808 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚262 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚169269903076352 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 4552982528 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 5368709120 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 4552982528 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚429 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 125829120 IBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚60 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚43407423 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚169269962129408 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚262 _step‚505 I@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚4876 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚20645904 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚43407423 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚168275925999029 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚41348068 BDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654602240 G@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚66 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚322 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚166792398766080 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚262 _runtime‚1659.400175422 DDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59053056 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 4654602240 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚41948160 H@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚665 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 $ _timestamp‚1.786660310869891e+09 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚43411718 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚19338065 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 4517085184 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚4295 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17104896 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚22765812 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 1571739264000 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚4293 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚4293 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚14173004 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 1571783299072 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚61478912 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 122905600 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 1483468845521 trainer/global_step‚6149 >5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5264 B9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚5264 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚41948160 SFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚64678912 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚322 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚168275867611601 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 5368709120 NADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚59053056 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚14173002 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 845178880 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚322 NADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚58387428 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 125829120 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚5164997 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚20643840 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 831258624 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚166792415805440 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 5494538240 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 5368709120 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 122905600 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚167783355449344 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 1486564731904 D=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚60 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17104896 NGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚64 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚44035072 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 5494538240 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 G@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚60 EDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚170571053325312 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 1497962374144 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚61478912 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 _step‚508 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚148980436035072 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 122306526 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚43747624 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚22939909 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 1497962374144 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚20807713 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚5204968 IBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚83 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚19485934 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚169570302732861 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 1494800083517 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 125829120 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 4552982528 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 4654602240 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 IDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4876 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 122905600 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚429 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 122905600 KBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚4876 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 5368709120 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 1497920429056 JBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚665 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚147396699947008 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 5494538240 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚170571112375296 M@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚59049984 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚169073132896256 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚322 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 E8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚43743335 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 845178880 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 IBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚60 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 1583811255808 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 5494538240 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚262 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚223 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚54523904 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 5494538240 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚169073150001152 PCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚59049984 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17104896 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚209 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚168075502649344 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 4517085184 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 5368709120 C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚4289 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚169073150001152 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚41945088 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 125829120 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚262 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚262 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 4654602240 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚169570361120241 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚19486017 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚14281047 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 831258624 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 4628521206 % _timestamp‚1.7866603237694955e+09 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚322 BDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚170571112375296 KDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚60 F?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚60 _runtime‚1672.303005119 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚170571053325312 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚322 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚20807713 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚41945088 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4289 D;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚5264 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚14281049 UHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚75167744 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚168075519688704 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 1497920429056 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚22939909 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 5368709120 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚322 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚20643840 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚4287 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 4552982528 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚147396679303168 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚22939911 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17104896 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚22939911 PCDeviceStatsMonitor.on_train_batch_start/allocat£#ò»Ïion.small_pool.freed‚20803426 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚148980511202816 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚4287qƒøöÂ'ý¼O PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚20807713 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚41945088 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4289 D;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚5264 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚14281049 UHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚75167744 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚168075519688704 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 1497920429056 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚22939909 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 5368709120 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚322 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚20643840 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚4287 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 4552982528 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚147396679303168 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚22939911 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17104896 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚22939911 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚20803426 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚148980511202816 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚4287 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚5204887 IDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚170571053325312 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 1497962374144 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚61478912 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 _step‚508 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚148980436035072 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 122306526 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚43747624 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚22939909 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 1497962374144 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚20807713 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚5204968 IBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚83 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚19485934 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚169570302732861 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 1494800083517 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 125829120 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 4552982528 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 4654602240 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 IDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4876 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 122905600 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚429 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 122905600 KBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚4876 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 5368709120 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 1497920429056 JBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚665 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚147396699947008 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 5494538240 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚170571112375296 M@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚59049984 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚169073132896256 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚322 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 E8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚43743335 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 845178880 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 IBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚60 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 1583811255808 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 5494538240 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚262 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚223 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚54523904 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 5494538240 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚169073150001152 PCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚59049984 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17104896 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚209 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚168075502649344 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 4517085184 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 5368709120 C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚4289 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚169073150001152 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚41945088 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 125829120 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚262 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚262 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 4654602240 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚169570361120241 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚19486017 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚14281047 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 831258624 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 4628521206 % _timestamp‚1.7866603237694955e+09 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚322 BDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚170571112375296 KDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚60 F?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚60 _runtime‚1672.303005119 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚170571053325312 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚322üš³×Ú2þ- _wandbruntime‚1672  _runtime‚1672ŠpšÇ&ÿÁM D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚43754470 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚169099472338944 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521206 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚429 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 122905600 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 1498193445888 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚20811006 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 5368709120 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚22943464 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 845178880 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 IBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚60 G@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚60 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚20643840 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 125829120 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚322 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚223 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 4552982528 H@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚665 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚19488955 G@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚71 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚19488884 NGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚69 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚41348068 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚262 D=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚60 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚170597665784832 NADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚59053056 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚14283254 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 1584051925504 _step‚509 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚20811006 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚4293 SFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚70970368 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚61478912 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚169099472338944 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 1498193445888 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚147419726675968 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 122905600 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚22943464 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚322 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚5205632 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚168101688442880 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 trainer/global_step‚6199 P5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5264 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚170597606731776 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚168101705482240 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 4552982528 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚41948160 EDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚4295 I@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚4876 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚262 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚43754470 B9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚5264 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚149003799245312 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 1584001598976 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17104896 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚169596719099009 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 4517085184 EDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654602240 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚322 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 125829120 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚169099455234048 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 831258624 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 _runtime‚1672.468671637 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚4293 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 1498151497728 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚169099455234048 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚5205701 K>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59053056 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17104896 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚43750175 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚43750175 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 1498151497728 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚209 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚50326528 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 1495072004197 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 5494538240 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 5368709120 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚22943462 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚170597665784832 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚22943462 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚149003728274944 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 5368709120 DDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚19488884 NGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚69 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚41348068 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚262 D=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚60 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚170597665784832 NADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚59053056 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚14283254 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 1584051925504 _step‚509 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚20811006 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚4293 SFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚70970368 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚61478912 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚169099472338944 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 1498193445888 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚147419726675968 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 122905600 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚22943464 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚322 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚5205632 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚168101688442880 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 trainer/global_step‚6199 P5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5264 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚170597606731776 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚168101705482240 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 4552982528 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚41948160 EDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚4295 I@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚4876 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚262 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚43754470 B9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚5264 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚149003799245312 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 1584001598976 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17104896 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚169596719099009 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 4517085184 EDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654602240 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚322 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 125829120 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚169099455234048 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 831258624 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 _runtime‚1672.468671637 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚4293 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 1498151497728 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚169099455234048 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚5205701 K>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59053056 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17104896 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚43750175 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚43750175 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 1498151497728 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚209 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚50326528 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 1495072004197 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 5494538240 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 5368709120 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚22943462 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚170597665784832 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚22943462 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚149003728274944 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 5368709120 DDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚43754470 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚169099472338944 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521206 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚429 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 122905600 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 1498193445888 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚20811006 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 5368709120 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚22943464 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 845178880 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 IBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚60 G@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚60 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚20643840 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360ýy!÷2- _wandbruntime‚1672  _runtime‚1672(·Wãs‚í 0loss/FRt2/assignment_loss‚0.6615311220947938 /loss/FRt1/detection_loss‚0.5032837538381914 &loss/total_loss‚2.2603591259586873 epoch‚15 $ _timestamp‚1.786660323939446e+09 _runtime‚1672.469320432 0loss/FRt1/assignment_loss‚0.6123344288927607 0loss/FRt2/detection_loss‚0.48320982113294103 trainer/global_step‚6199 _step‚5102ͳ3xƒò 0loss/FRt2/assignment_loss‚0.6615311220947938 /loss/FRt1/detection_loss‚0.5032837538381914 &loss/total_loss‚2.2603591259586873 epoch‚15 $ _timestamp‚1.786660323939446e+09 _runtime‚1672.469320432 0loss/FRt1/assignment_loss‚0.6123344288927607 0loss/FRt2/detection_loss‚0.48320982113294103 trainer/global_step‚6199 _step‚510þ¹‰Ø–„‚(:‹ ë‹ùÓó ”á gpu.0.gpu‚60.0 gpu.0.memory‚11, gpu.0.memoryAllocated‚7.0313262939453125) gpu.0.memoryAllocatedBytes‚ 6039863296 gpu.0.temp‚34.0 gpu.0.powerWatts‚120.113' gpu.0.enforcedPowerLimitWatts‚500.0 gpu.0.powerPercent‚24.0226 gpu.0.smClock‚1410 gpu.0.memoryClock‚1593$ gpu.0.correctedMemoryErrors‚1955# gpu.0.uncorrectedMemoryErrors‚0ˆÁàŠ…‚(:ÿ ë‹ùÓ¯î±ä disk.drbd2165.out‚0" proc.memory.rssMB‚ 2376.8046875 network.sent‚ 8.297643e+06 disk.sda.in‚16339.5 disk.dm-1.in‚16339.5 disk.dm-1.out‚ 33.26953125 disk.sda2.out‚0$ disk./.usageGB‚487.4100036621094 disk.sda.out‚ 33.26953125 disk.sda1.in‚0 disk.sda1.out‚0 disk.dm-0.in‚43098.77734375$ memory_percent‚66.78028106689453* proc.memory.percent‚14.506864547729492 disk.sda3.in‚16339.5( disk.nvme0c0n1.out‚53208.17431640625" disk.nvme0n1.in‚43098.77734375& disk.nvme0n1.out‚53208.17431640625# disk.dm-0.out‚53208.17431640625& proc.memory.availableMB‚ 5442.71875 cpu‚27.037343405528947 proc.cpu.threads‚17 disk.sda2.in‚0 network.recv‚ 3.343662e+06* disk./.usagePercent‚32.710494309997024 disk.sda3.out‚ 33.26953125$ disk.nvme0c0n1.in‚43098.77734375 disk.drbd2165.in‚0S@WY2†- _wandbruntime‚1685  _runtime‚1685¹%´Ç‡Á & lr-AdamW/pg1‚0.0011938483951475741 & lr-AdamW/pg2‚0.0011938483951475741 trainer/global_step‚6249 % _timestamp‚1.7866603365885932e+09 _runtime‚1685.118652257 _step‚511˜¶8(ˆÆ & lr-AdamW/pg1‚,±O ³0.0011938483951475741 & lr-AdamW/pg2‚0.0011938483951475741 trainer/global_step‚6249 % _timestamp‚1.7866603365885932e+09 _runtime‚1685.118652257 _step‚511ÿ¶ìÚâ2‰- _wandbruntime‚1685  _runtime‚1685¢¨Ù»'ŠµO ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 1509549139968 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 125829120 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚23117559 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚14391297 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚150145078027776 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚262 F>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚20972815 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚4289 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚5245520 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 845178880 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚169384792326144 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚322 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚20968528 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 125829120 # _timestamp‚1.78666033658977e+09 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚429 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 1509507194880 PCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚58387380 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚14391299 D;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚5264 PIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚85 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚170891212607649 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 4628521206 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 PCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚59049984 KBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚4876 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚148549068587008 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚148549047943168 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 5494538240 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 1509507194880 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 5368709120 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚44090376 BDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4876 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 122905600 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚209 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚170891154220269 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 5494538240 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚262 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚23117561 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚19636904 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚61478912 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚322 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚170389266890752 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚171898756980736 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚20643840 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚23117561 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 4552982528 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 122905600 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 5494538240 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 M@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚59049984 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚171898756980736 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚223 F?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚60 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 4654602240 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 4552982528 UHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚73070592 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚262 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚4287 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 4517085184 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 122306526 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 5368709120 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17104896 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚171898816030720 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚41945088 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚20972815 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚19636817 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚169384809365504 @7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚5264 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 831258624 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚322 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 1506403242145 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 5368709120 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 125829120 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚322 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17104896 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚41348020 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 _runtime‚1685.123917389 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 4654602240 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚150145151098368 IDeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4289 JBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚665 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 _step‚512 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚20968528 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 KDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚60 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚41945088(5ä˜À'‹ºO SGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚5245520 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 845178880 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚169384792326144 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚322 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚20968528 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 125829120 # _timestamp‚1.78666033658977e+09 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚429 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 1509507194880 PCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚58387380 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚14391299 D;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚5264 PIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚85 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚170891212607649 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 4628521206 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 PCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚59049984 KBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚4876 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚148549068587008 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚148549047943168 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 5494538240 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 1509507194880 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 5368709120 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚44090376 BDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4876 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 122905600 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚209 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚170891154220269 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 5494538240 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚262 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚23117561 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚19636904 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚61478912 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚322 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚170389266890752 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚171898756980736 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚20643840 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚23117561 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 4552982528 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 122905600 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 5494538240 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 M@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚59049984 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚171898756980736 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚223 F?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚60 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 4654602240 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 4552982528 UHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚73070592 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚262 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚4287 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 4517085184 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 122306526 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 5368709120 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17104896 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚171898816030720 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚41945088 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚20972815 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚19636817 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚169384809365504 @7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚5264 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 831258624 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚322 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 1506403242145 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 5368709120 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 125829120 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚322 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17104896 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚41348020 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 _runtime‚1685.123917389 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 4654602240 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚150145151098368 IDeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4289 JBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚665 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 _step‚512 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚20968528 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 KDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚60 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚41945088 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 1509549139968 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 125829120 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚23117559 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚14391297 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚150145078027776 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚262 F>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚20972815 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚4289€*uq=2Œ- _wandbruntime‚1685  _runtime‚1685;xwÇ&ÁM ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚169410978119680 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 4517085184 trainer/global_step‚6249 _runtime‚1685.290976286 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚23121112 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚44092927 I@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚4876 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚41348068 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 5494538240 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚322 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚262 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 5494538240 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 1509780211712 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚148572115959808 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 1509738263552 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚322 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚171925369440256 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17104896 NADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚59053056 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 4552982528 NGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚75 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚150168445760000 D=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚60 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚14393502 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚4295 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚56617984 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 4654602240 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚171925369440256 EDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 >5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5264 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚20976108 G@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚60 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 1506592466737 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 5494538240 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 5368709120 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 DDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚44097222 H@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚665 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 5368709120 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚170415572123648 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654602240 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 1506633814805 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 125829120 PDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59053056 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17104896 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚41948160 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚4293 G@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚77 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 122306526 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 5368709120 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 122905600 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚169410995159040 EDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521206 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚23121114 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚19639800 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 831258624 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚322 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚262 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚41948160 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 122905600 B9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚5264 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚23121114 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚23121112 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 1596329800192 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 125829120 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚19639877 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚148572095315968 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 1596273182208 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚61478912èbÕ¥vŽÆM HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 4654602240 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚171925369440256 EDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 >5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5264 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚20976108 G@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚60 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 1506592466737 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 5494538240 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 5368709120 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 DDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚44097222 H@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚665 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 5368709120 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚170415572123648 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654602240 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 1506633814805 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 125829120 PDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59053056 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17104896 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚41948160 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚4293 G@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚77 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 122306526 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 5368709120 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 122905600 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚169410995159040 EDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521206 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚23121114 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚19639800 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 831258624 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚322 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚262 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚41948160 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 122905600 B9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚5264 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚23121114 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚23121112 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 1596329800192 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 125829120 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚19639877 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚148572095315968 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 1596273182208 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚61478912 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚169410978119680 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 4517085184 trainer/global_step‚6249 _runtime‚1685.290976286 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚23121112 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚44092927 I@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚4876 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚41348068 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 5494538240 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚322 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚262 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 5494538240 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 1509780211712 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚148572115959808 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 1509738263552 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚322 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚171925369440256 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17104896 NADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚59053056 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 4552982528 NGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚75 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚150168445760000 D=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚60 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚14393502 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚4295 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚56617984^èw2- _wandbruntime‚1685  _runtime‚1685Ÿ ÃQrì /loss/FRt1/detection_loss‚0.5365945946706222 trainer/global_step‚6249 0loss/FRt1/assignment_loss‚0.6766873103635267 /loss/FRt2/assignment_loss‚0.637121976592468 0loss/FRt2/detection_loss‚0.46919482329974765 &loss/total_loss‚2.3195987049263644 epoch‚15 $ _timestamp‚1.786660336761639e+09 _runtime‚1685.291549429 _step‚514~é/Ýw‘ñ /loss/FRt2/assignment_loss‚0.637121976592468 0loss/FRt2/detection_loss‚0.46919482329974765 &loss/total_loss‚2.3195987049263644 epoch‚15 $ _timestamp‚1.786660336761639e+09 _runtime‚1685.291549429 _step‚514 /loss/FRt1/detection_loss‚0.5365945946706222 trainer/global_step‚6249 0loss/FRt1/assignment_loss‚0.6766873103635267‚œÊ()•’‚(:Š ú‹ùÓ†Ïûà gpu.0.gpu‚27.0 gpu.0.memory‚3, gpu.0.memoryAllocated‚7.0313262939453125) gpu.0.memoryAllocatedBytes‚ 6039863296 gpu.0.temp‚35.0 gpu.0.powerWatts‚163.132' gpu.0.enforcedPowerLimitWatts‚500.0 gpu.0.powerPercent‚32.6264 gpu.0.smClock‚1410 gpu.0.memoryClock‚1593$ gpu.0.correctedMemoryErrors‚1955# gpu.0.uncorrectedMemoryErrors‚0¤pU•“‚(:Š ú‹ùÓ‚ûÿâ disk.sda3.out‚ 33.44921875$ disk.nvme0c0n1.in‚43120.00390625( disk.nvme0c0n1.out‚53252.18798828125 network.sent‚ 8.367571e+06 disk.sda.out‚ 33.44921875 disk.sda2.in‚0 disk.sda2.out‚0& disk.nvme0n1.out‚53252.18798828125 disk.drbd2165.out‚0 disk.dm-0.in‚43120.00390625# disk.dm-0.out‚53252.18798828125( disk./.usagePercent‚32.7104769014847 disk.sda.in‚ 16568.21875 disk.dm-1.out‚ 33.44921875 disk.sda3.in‚ 16568.21875$ memory_percent‚66.77947044372559( proc.memory.availableMB‚ 5442.8515625" proc.memory.rssMB‚ 2376.8046875* proc.memory.percent‚14.506864547729492$ disk./.usageGB‚487.4097442626953 disk.sda1.in‚0 cpu‚27.03850782678317 proc.cpu.threads‚17 network.recv‚ 3.370452e+06 disk.sda1.out‚0" disk.nvme0n1.in‚43120.00390625 disk.dm-1.in‚ 16568.21875 disk.drbd2165.in‚06a(¦2”- _wandbruntime‚1697  _runtime‚1697M£Ø§Ç•Á & lr-AdamW/pg1‚0.0011889890494956588 & lr-AdamW/pg2‚0.0011889890494956588 trainer/global_step‚6299 % _timestamp‚1.7866603489779427e+09 _runtime‚1697.508014466 _step‚515ÖuICÌ–Æ & lr-AdamW/pg1‚0.0011889890494956588 & lr-AdamW/pg2‚0.0011889890494956588 trainer/global_step‚6299 % _timestamp‚1.7866603489779427e+09 _runtime‚1697.508014466 _step‚515ƒB±ì2—- _wandbruntime‚1697  _runtime‚1697‰I#8¼'˜¶O SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 4552982528 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17104896 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚223 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 831258624 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 5494538240 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 5368709120 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚41348020 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 5494538240 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 5494538240 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚262 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 1521093960704 IDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚173226460636160 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 1608338808320 KDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚60 $ _timestamp‚1.786660348978997e+09 IDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4289 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚23295211 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚171705366675456 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 845178880 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 G>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4876 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚23295211 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 _step‚516 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚173226460636160 PCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚58387380 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚170694099042304 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 125829120 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚151309707159040 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚149701437227008 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚171705383780352 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚41945088 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 125829120 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚322 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚262 trainer/global_step‚6299 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚21137917 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚5286184 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚61478912 IBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚60 C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚4289 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚21137917 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚41945088 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚14501547 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚151309776035328 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚322 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17104896 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚173226519686144 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚44433128 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚14501549 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚5286114 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 1517923704733 @7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚5264 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚21133630 PIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚70 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚172212064095057 M@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚59049984 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚429 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚21133630 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚172212005707677 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚170694082002944 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 122905600 D;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚5264 UHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚68876288 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 1608290575872 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚23295209 IBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚72 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 1517965052753 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚262 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 1521135905792 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 122905600 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚209 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 5368709120 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 4552982528 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚20643840 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 125829120 E8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚44428839 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 1521135905792 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚23295209 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚4287 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚19787733'µ Á'™»O PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚21133630 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚172212005707677 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚170694082002944 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 122905600 D;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚5264 UHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚68876288 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 1608290575872 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚23295209 IBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚72 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 1517965052753 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚262 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 1521135905792 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 122905600 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚209 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 5368709120 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 4552982528 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚20643840 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 125829120 E8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚44428839 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 1521135905792 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚23295209 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚4287 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚19787733 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 4552982528 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17104896 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚223 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 831258624 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 5494538240 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 5368709120 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚41348020 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 5494538240 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 5494538240 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚262 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 1521093960704 IDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚173226460636160 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 1608338808320 KDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚60 $ _timestamp‚1.786660348978997e+09 IDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4289 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚23295211 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚171705366675456 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 845178880 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 G>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4876 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚23295211 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 _step‚516 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚173226460636160 PCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚58387380 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚170694099042304 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 125829120 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚151309707159040 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚149701437227008 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚171705383780352 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚41945088 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 125829120 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚322 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚262 trainer/global_step‚6299 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚21137917 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚5286184 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚61478912 IBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚60 C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚4289 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚21137917 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚41945088 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚14501547 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚151309776035328 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚322 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17104896 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚173226519686144 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚44433128 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚14501549 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚5286114 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 1517923704733 @7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚5264 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚21133630 PIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚70 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚172212064095057 M@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚59049984 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚429„Àâý’2š- _wandbruntime‚1697  _runtime‚1697W]m_]›ÀM T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚173253073095680 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 4552982528 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚61478912 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚14503754 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚41948160 E8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 4552982528 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 1608587301376 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 1521325029376 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 122905600 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚209 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 5368709120 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚44439974 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚52423680 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 5494538240 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚223 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚44435679 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚21136917 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 5494538240 >6DeviceStatsMonitor.on_train_ba*ñ€Æitch_end/segment.all.peak‚322 trainer/global_step‚6299 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚20643840 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚171731706118144 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚170720284835840 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 122306526 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚44435679 NADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚59053056 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654602240 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚23298764 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚262 G@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚60 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚5286961 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚322 K>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59053056 DDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚19790643 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 5494538240 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17104896 PDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521206 >5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5264 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚21141210 I@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚4876 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚322 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 5368709120 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚4295 NGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚70 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 831258624 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚172238422073825 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 125829120 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚23298762 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚173253073095680 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚23298762 H@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚665 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚19790715 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚149724463955968 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚322 IBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚60 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚4293 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚149724484599808 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 NADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚58387428 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 125829120 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚5286891 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 G@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚72 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚23298764 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚171731706118144 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚44439974 _step‚517 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17104896 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 122905600 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚173253014042624 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 1521325029376 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 1608534877696 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 1518154277345 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚41948160 D=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚60ˆ–¯÷Ë&œÅM VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚14503754 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚41948160 E8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 4552982528 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 1608587301376 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 1521325029376 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 122905600 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚209 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 5368709120 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚44439974 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚52423680 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 5494538240 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚223 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚44435679 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚21136917 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 5494538240 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚322 trainer/global_step‚6299 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚20643840 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚171731706118144 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚170720284835840 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 122306526 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚44435679 NADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚59053056 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654602240 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚23298764 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚262 G@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚60 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚5286961 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚322 K>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59053056 DDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚19790643 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 5494538240 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17104896 PDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521206 >5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5264 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚21141210 I@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚4876 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚322 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 5368709120 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚4295 NGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚70 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 831258624 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚172238422073825 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 125829120 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚23298762 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚173253073095680 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚23298762 H@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚665 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚19790715 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚149724463955968 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚322 IBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚60 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚4293 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚149724484599808 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 NADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚58387428 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 125829120 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚5286891 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 G@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚72 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚23298764 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚171731706118144 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚44439974 _step‚517 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17104896 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 122905600 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚173253014042624 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 1521325029376 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 1608534877696 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 1518154277345 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚41948160 D=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚60 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚173253073095680 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 4552982528 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚61478912…;#—ˆ2- _wandbruntime‚1697  _runtime‚1697?¸^qžë 0loss/FRt2/assignment_loss‚0.7590367462576363 .loss/FRt1/detection_loss‚0.540891490977666 /loss/FRt2/detection_loss‚0.4823863517457502 %loss/total_loss‚2.537603740118393 0loss/FRt1/assignment_loss‚0.7552891511373406 epoch‚15 trainer/global_step‚6299 % _timestamp‚1.7866603491425552e+09 _runtime‚1697.672369336 _step‚518ëÄÚvŸð epoch‚15 trainer/global_step‚6299 % _timestamp‚1.7866603491425552e+09 _runtime‚1697.672369336 _step‚518 0loss/FRt2/assignment_loss‚0.7590367462576363 .loss/FRt1/detection_loss‚0.540891490977666 /loss/FRt2/detection_loss‚0.4823863517457502 %loss/total_loss‚2.537603740118393 0loss/FRt1/assignment_loss‚0.7552891511373406†Iì*2 - _wandbruntime‚1709  _runtime‚1709ºv®¦Ç¡Á & lr-AdamW/pg1‚0.0011841015083974124 & lr-AdamW/pg2‚0.0011841015083974124 trainer/global_step‚6349 % _timestamp‚1.7866603614489057e+09 _runtime‚1709.978921031 _step‚519·îâÌ¢Æ % _timestamp‚1.7866603614489057e+09 _runtime‚1709.978921031 _step‚519 & lr-AdamW/pg1‚0.0011841015083974124 & lr-AdamW/pg2‚0.0011841015083974124 trainer/global_step‚6349‡=›³M2£- _wandbruntime‚1709  _runtime‚1709ÿü„‡½'¤·O TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚14611797 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚173021483565056 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚41945088 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚21298732 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚262 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 1532680726528 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 _runtime‚1709.980212351 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 125829120 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚429 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 % _timestamp‚1.7866603614498854e+09 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚23472859 G>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4876 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 4654602240 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚174554223341568 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚262 M@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚59049984 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚173021500669952 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17104896 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 1532722671616 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚41945088 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 122905600 IDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚174554164291584 F?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚60 F>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚173021483565056 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚19938604 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚23472861 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚21303019 BDeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4289 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚5326807 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 1529485515341 KDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚60 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚223 PCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚59049984 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 4552982528 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 UHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚66779136 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚173532915582465 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚322 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 845178880 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚322 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚174554223341568 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚44775880 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚209 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 5494538240 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 125829120 trainer/global_step‚6349 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚174554164291584 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚152474354977280 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0ç­°–² ¥¼O L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 4654602240 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚174554223341568 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚262 M@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚59049984 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚173021500669952 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17104896 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 1532722671616 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚41945088 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 122905600 IDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚174554164291584 F?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚60 F>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚173021483565056 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚19938604 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚23472861 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚21303019 BDeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4289 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚5326807 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 1529485515341 KDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚60 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚223 PCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚59049984 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 4552982528 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 UHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚66779136 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚173532915582465 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚322 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 845178880 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚322 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚174554223341568 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚44775880 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚209 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 5494538240 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 125829120 trainer/global_step‚6349 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚174554164291584 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚152474354977280 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚14611797 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚173021483565056 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚41945088 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚21298732 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚262 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 1532680726528 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 _runtime‚1709.980212351 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 125829120 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚429 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 % _timestamp‚1.7866603614498854e+09 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚23472859 G>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4876 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0ˆ†ÔȆ–¦‚(:‹ ‰ŒùÓÅÁ©á gpu.0.gpu‚62.0 gpu.0.memory‚11, gpu.0.memoryAllocated‚7.0313262939453125) gpu.0.memoryAllocatedBytes‚ 6039863296 gpu.0.temp‚36.0 gpu.0.powerWatts‚218.977' gpu.0.enforcedPowerLimitWatts‚500.0 gpu.0.powerPercent‚43.7954 gpu.0.smClock‚1410 gpu.0.memoryClock‚1593$ gpu.0.correctedMemoryErrors‚1955# gpu.0.uncorrectedMemoryErrors‚0e%Ó‘•§‚(:Š ‰ŒùÓÕ‘„ä disk.dm-1.in‚ 16568.21875 disk.drbd2165.in‚0 cpu‚27.03943235899679% disk./.usageGB‚487.40699005126953 disk.sda3.in‚ 16568.21875 disk.sda3.out‚ 33.66796875# disk.nvme0c0n1.in‚ 43123.5390625 disk.sda.out‚ 33.66796875 disk.sda1.in‚0 disk.sda2.out‚0$ memory_percent‚66.77801609039307 network.sent‚ 8.431464e+06 network.recv‚ 3.391234e+06 disk.drbd2165.out‚0 disk.dm-0.in‚ 43123.5390625" proc.memory.rssMB‚ 2376.8046875* proc.memory.percent‚14.506864547729492 disk.sda1.out‚0& disk.nvme0n1.out‚53259.28857421875 disk.dm-1.out‚ 33.66796875# disk.dm-0.out‚53259.28857421875) proc.memory.availableMB‚ 5443.08984375 proc.cpu.threads‚17) disk./.usagePercent‚32.71029206404498( disk.nvme0c0n1.out‚53259.28857421875 disk.sda.in‚ 16568.21875 disk.sda2.in‚0! disk.nvme0n1.in‚ 43123.5390625Øfl¡2¨- _wandbruntime‚1710  _runtime‚1710Uì*Ç&©ÁM ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚48229376 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚172029574512640 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚21302019 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 125829120 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚14614002 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 NADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚58387428 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 1532911795200 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚173047805902848 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚209 G@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚60 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚5327645 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 5368709120 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 >5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5264 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 1532953743360 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚429 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚20643840 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 845178880 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 H@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚665 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚4293 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17104896 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚41948160 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 1620860196352 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚172029557473280 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 125829120 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 122905600 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚174580776751104 NGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚67 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚152497713436160 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 5494538240 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 K>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59053056 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚44782726 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚23476412 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 1620811966976 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚262 _step‚521 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17104896 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚173559273561233 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 5494538240 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 5494538240 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 D6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚322 SFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚68873216 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 122306526 EDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654602240 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 1532953743360 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 122905600 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 1529716087953 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚4295 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚21306312 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 5368709120 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚174580776751104 NADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚59053056 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚173559331948661 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 5368709120 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 125829120 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚322 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚23476414 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 4552982528 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚173047823007744 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 1532911795200 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚150876853239808 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 4552982528 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚23476414 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521206 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 4517085184 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚173047823007744 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚173047805902848 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚44778431 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚19941647 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚150876832595968 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 D=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚60 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚44778431 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚21306312 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚21302019 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚322 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 IBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚60 trainer/global_step‚6349 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚223 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 1529757436021 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚262 % _timestamp‚1.7866603616151524e+09 _runtime‚1710.145746918 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚4293 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚41948160 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚61478912 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 BDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 125829120 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚14614002 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 NADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚58387428 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 1532911795200 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚173047805902848 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚209 G@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚60 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚5327645 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 5368709120 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 >5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5264 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 1532953743360 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚429 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚20643840 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 845178880 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 H@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚665 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚4293 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17104896 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚41948160 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 1620860196352 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚172029557473280 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 125829120 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 122905600 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚174580776751104 NGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚67 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚152497713436160 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 5494538240 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 K>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59053056 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚44782726 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚23476412 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 1620811966976 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚262 _step‚521 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17104896 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚173559273561233 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 5494538240 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 5494538240 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 D6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚322 SFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚68873216 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 122306526 EDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654602240 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 1532953743360 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 122905600 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 1529716087953 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚4295 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚21306312 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 5368709120 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚174580776751104 NADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚59053056 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚173559331948661 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 5368709120 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 125829120 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚322 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚23476414 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 4552982528 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚173047823007744 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 1532911795200 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚150876853239808 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 4552982528 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚23476414 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521206 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 4517085184 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚173047823007744 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚173047805902848 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚44778431 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚19941647 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚150876832595968 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 D=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚60 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚44778431 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚21306312 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚21302019 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚322 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 IBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚60 trainer/global_step‚6349 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚223 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 1529757436021 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚262 % _timestamp‚1.7866603616151524e+09 _runtime‚1710.145746918 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚4293 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚41948160 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚61478912 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 BŸc2±- _wandbruntime‚1722  _runtime‚1722y&cŒ ²·O WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 125829120 IDeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4289 IBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚78 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 125829120 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚20089924 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚5367875 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚175881926996992 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 4517085184 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚262 F?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚60 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚41348020 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 122306526 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 5368709120 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 M@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚59049984 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚322 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚41945088 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚223 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚52426752 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚173312661356544 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚322 IBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚60 trainer/global_step‚6399 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚14722047 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 @7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚5264 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚23650509 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚174853767069873 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚174853708682493 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚153638980623872 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚173312678395904 F@DeviceStatsMonitor.on_train_batch5Ñwž1_start/segment.large_pool.freed‚0 _runtime‚1722.453878434 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 1544267492352 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚21468121 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚152006153863168 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚61478912 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚23650511 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 4628521206 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚23650511 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 1544309437440 JBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚665 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚152006174507008 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚262 G>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4876 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚175881867947008 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 5494538240 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 125829120 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 1544267492352 PCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚59049984 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 1632879187456 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 PCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚58387380 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚322 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17104896 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚23650509 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚20089846 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 1632826760704 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚322 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚14722049 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚174337617559552 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚175881867947008 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚41945088 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚21463834 UHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚73070592 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚174337617559552 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 4552982528 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 PIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚76 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 831258624 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚262 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 5368709120 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 122905600 KBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚4876 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚153639053694464 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚21463834 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚175881926996992 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚429 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚209 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚20643840 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 F>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 122905600 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 1541047325949 % _timestamp‚1.7866603739198487e+09 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚174337600454656 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚4287 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚174337600454656 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚45118632 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 5494538240 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 4654602240 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 845178880 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚21468121 BDeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4289 IBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚78 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 125829120 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚20089924 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚5367875 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚175881926996992 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 4517085184 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚262 F?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚60 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚41348020 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 122306526 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 5368709120 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 M@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚59049984 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚322 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚41945088 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚223 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚52426752 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚173312661356544 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚322 IBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚60 trainer/global_step‚6399 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚14722047 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 @7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚5264 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚23650509 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚174853767069873 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚174853708682493 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚153638980623872 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚173312678395904 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 _runtime‚1722.453878434 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 1544267492352 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚21468121 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚152006153863168 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚61478912 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚23650511 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 4628521206 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚23650511 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 1544309437440 JBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚665 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚152006174507008 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚262 G>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4876 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚175881867947008 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 5494538240 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 125829120 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 1544267492352 PCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚59049984 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 1632879187456 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 PCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚58387380 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚322 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17104896 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚23650509 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚20089846 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 1632826760704 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚322 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚14722049 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚174337617559552 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚175881867947008 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚41945088 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚21463834 UHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚73070592 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚174337617559552 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 4552982528 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 PIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚76 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 831258624 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚262 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 5368709120 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 122905600 KBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚4876 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚153639053694464 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚21463834 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚175881926996992 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚429 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚209 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚20643840 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 F>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 122905600 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 1541047325949 % _timestamp‚1.7866603739198487e+09 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚174337600454656 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚4287Œ´§4¼2´- _wandbruntime‚1722  _runtime‚1722V ¶xÆ&µÀM JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚41348068 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚21471414 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17104896 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚20092944 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚45121183 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 NADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚59053056 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚4293 BDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚45125478 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚14724252 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚174880125048641 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 5494538240 >5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5264 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚41948160 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521206 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 5494538240 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚322 NGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚59 _runtime‚1722.610866599 EDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654602240 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚5368631 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 5368709120 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚23654062 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚175908421353472 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚174363939897344 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚153662342425088 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 5368709120 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚23654064 B9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚5264 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚209 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 845178880 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 IBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚60 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 122905600 I@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚4876 SFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚64678912 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚322 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚173338847150080 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 125829120 K>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59053056 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17104896 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 1544540509184 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 831258624 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚44035072 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 125829120 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 125829120 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚175908480406528 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚14724254 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚262 _step‚525 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚429 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 1541277898561 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚322 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚262 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚262 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 4552982528 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚21471414 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚20092883 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚174880183436069 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚21467121 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚45125478 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚4293 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 4552982528 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 1544498561024 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 DDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654602240 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚5368631 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 5368709120 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚23654062 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚175908421353472 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚174363939897344 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚153662342425088 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 5368709120 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚23654064 B9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚5264 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚209 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 845178880 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 IBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚60 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 122905600 I@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚4876 SFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚64678912 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚322 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚173338847150080 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 125829120 K>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59053056 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17104896 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 1544540509184 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 831258624 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚44035072 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 125829120 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 125829120 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚175908480406528 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚14724254 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚262 _step‚525 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚429 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 1541277898561 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚322 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚262 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚262 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 4552982528 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚21471414 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚20092883 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚174880183436069 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚21467121 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚45125478 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚4293 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 4552982528 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 1544498561024 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 DDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚45121183 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 NADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚59053056 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚4293 BDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚45125478 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚14724252 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚174880125048641 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 5494538240 >5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5264 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚41948160 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521206 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 5494538240 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚322 NGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚59 _runtime‚1722.610866599 EDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚177604576543744 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚429 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚46135296 G>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4876 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17104896 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚41945088 D;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚5264 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚262 trainer/global_step‚6449 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 122306526 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 5494538240 PCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚59049984 C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚4289 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚41945088 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚176559584021227 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚322 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚262 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 5368709120 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚176044671664128 IDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 1649209962496 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 F>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17104896 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 1559963929600 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 F?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚60 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 5494538240 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 125829120 @7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚5264 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚23888765 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 122905600 IBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚60 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚21655415 IBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚76 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚5417327 PIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚74 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 4517085184 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚176044654559232 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚23888767 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚5417253 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚209 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚155107002773504 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚322 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 1556716456683 _step‚529 JBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚665 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚21651128 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 5368709120 E8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚45539893 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚45544182 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚223 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚175002850525184 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚177604635593728 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚177604576543744 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 1559921984512 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4289 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 125829120 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚262 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 1649256097792 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 5368709120 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 4552982528 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚176044654559232 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚20290164 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0pM"GÂ'ƼO PCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚59049984 C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚4289 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚41945088 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚176559584021227 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚322 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚262 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 5368709120 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚176044671664128 IDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 1649209962496 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 F>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17104896 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 1559963929600 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 F?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚60 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 5494538240 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 125829120 @7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚5264 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚23888765 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 122905600 IBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚60 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚21655415 IBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚76 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚5417327 PIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚74 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 4517085184 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚176044654559232 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚23888767 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚5417253 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚209 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚155107002773504 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚322 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 1556716456683 _step‚529 JBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚665 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚21651128 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 5368709120 E8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚45539893 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚45544182 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚223 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚175002850525184 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚177604635593728 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚177604576543744 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 1559921984512 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4289 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 125829120 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚262 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 1649256097792 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 5368709120 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 4552982528 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚176044654559232 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚20290164 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 125829120 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 4654602240 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 1559963929600 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 831258624 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 5494538240 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚176559525633847 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚153457813454848 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 % _timestamp‚1.7866603925960891e+09 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚4287 BDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚177604576543744 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚429 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚46135296 G>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4876 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17104896 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚41945088 D;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚5264 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚262 trainer/global_step‚6449 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 122306526 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 5494538240‘w·2Ç- _wandbruntime‚1741  _runtime‚1741à…”ÈÁM E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚23892320 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 125829120 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 NGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚79 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚429 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚322 IBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚60 % _timestamp‚1.7866603927642891e+09 EDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 trainer/global_step‚6449 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚23892320 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚176070994001920 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 4552982528 Ð[$j3!K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚45551028 H@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚665 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚262 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 122905600 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚4293 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 831258624 D=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚60 >5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5264 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 1560153053184 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17104896 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚175029036318720 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 5368709120 PDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521206 _step‚530 DDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654602240 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚21658708 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 1649500077056 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚14875116 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚175029053358080 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚322 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚262 _runtime‚1741.294641989 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 4552982528 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚61478912 NADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚59053056 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚41948160 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 125829120 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚176070976897024 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚14875118 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 5494538240 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 125829120 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚45551028 I@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚4876 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 4517085184 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚45546733 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚23892318 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚5418183 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚155130296225792 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 5368709120 K>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59053056 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17104896 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚176070976897024 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚20293220 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚209 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚21654415 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚153480840183808 E6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚322 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚4293 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚155130360904704 SFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚64678912 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚41948160 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 1560153053184 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚223 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 1556947029343 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚262 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚177631129950208 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚44035072 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚176586000387423 G@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚60 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 122905600 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 1649456041984 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚177631189003264 B9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚5264 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 ôMéÌ&ÉÆM >5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5264 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 1560153053184 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17104896 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚175029036318720 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 5368709120 PDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521206 _step‚530 DDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654602240 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚21658708 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 1649500077056 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚14875116 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚175029053358080 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚322 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚262 _runtime‚1741.294641989 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 4552982528 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚61478912 NADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚59053056 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚41948160 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 125829120 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚176070976897024 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚14875118 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 5494538240 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 125829120 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚45551028 I@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚4876 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 4517085184 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚45546733 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚23892318 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚5418183 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚155130296225792 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 5368709120 K>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59053056 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17104896 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚176070976897024 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚20293220 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚209 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚21654415 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚153480840183808 E6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚322 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚4293 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚155130360904704 SFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚64678912 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚41948160 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 1560153053184 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚223 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 1556947029343 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚262 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚177631129950208 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚44035072 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚176586000387423 G@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚60 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 122905600 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 1649456041984 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚177631189003264 B9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚5264 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚23892320 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 125829120 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 NGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚79 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚429 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚322 IBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚60 % _timestamp‚1.7866603927642891e+09 EDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 trainer/global_step‚6449 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚23892320 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚176070994001920 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 4552982528 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚45551028 H@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚665 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚262 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 122905600 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚4293 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 831258624 D=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚60’ƒƒûc2Ê- _wandbruntime‚1741  _runtime‚1741™€®rËì trainer/global_step‚6449 _runtime‚1741.295196787 /loss/FRt2/detection_loss‚0.5148050181220817 $ _timestamp‚1.786660392765373e+09 _step‚531 0loss/FRt1/assignment_loss‚0.6729303213136305 0loss/FRt2/assignment_loss‚0.5929549534231999 /loss/FRt1/detection_loss‚0.4776279569486154 &loss/total_loss‚2.2583182498075276 epoch‚16ï}4ÕwÌñ &loss/total_loss‚2.2583182498075276 epoch‚16 trainer/global_step‚6449 _runtime‚1741.295196787 /loss/FRt2/detection_loss‚0.5148050181220817 $ _timestamp‚1.786660392765373e+09 _step‚531 0loss/FRt1/assignment_loss‚0.6729303213136305 0loss/FRt2/assignment_loss‚0.5929549534231999 /loss/FRt1/detection_loss‚0.4776279569486154“çÛ2Í- _wandbruntime‚1753  _runtime‚1753JÙŒÆÎÀ & lr-AdamW/pg1‚0.0011692728605346172 & lr-AdamW/pg2‚0.0011692728605346172 trainer/global_step‚6499 $ _timestamp‚1.786660404957345e+09 _runtime‚1753.487364072 _step‚532ªw½]ËÏÅ trainer/global_step‚6499 $ _timestamp‚1.786660404957345e+09 _runtime‚1753.487364072 _step‚532 & lr-AdamW/pg1‚0.0011692728605346172 & lr-AdamW/pg2‚0.0011692728605346172”øÏc=2Ð- _wandbruntime‚1753  _runtime‚1753àÕ,Þ½'Ñ·O BDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4876 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚178932339249152 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚154610161451008 IBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚60 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚154610182094848 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 1661528824832 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚44038144 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚262 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚262 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚177360771448832 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚429 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 1568236919271 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚45886934 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚21820517 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚20441711 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚223 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚5458479 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 845178880 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚322 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚322 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 122306526 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚14983163 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚24066415 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 831258624 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 5368709120 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 5368709120 @7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚5264 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚178932339249152 M@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚59049984 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 1571550695424 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚14983161 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚322 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 KBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚4876 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 _step‚533 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 4552982528 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 1571550695424 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4289 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚61478912 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 4517085184 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚156271646237696 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚177360771448832 IBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚71 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 5494538240 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚322 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 4654602240 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚178932280199168 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚4289 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚4287 PCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚58387380 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 4628521206 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚21816230 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚41945088 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 4552982528 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚20441640 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 1661484786688 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 KDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚60 trainer/global_step‚6499 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚177360788553728 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚176312140201984 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚41348020 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 5368709120 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 125829120 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 125829120 E8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚45882645 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚21820517 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚178932280199168 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 4654602240 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 125829120 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚262 % _timestamp‚1.7866604049582705e+09 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17104896 PCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚59049984 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚177880435508635 F>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚209 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚20643840 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 _runtime‚1753.491899252 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17104896 PIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚69 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 1568278267291 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 5494538240 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚24066417 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 1571508750336 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 122905600 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚21816230 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 F?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚60 IDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4876 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 Y Û„ÛEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚178932339249152 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚154610161451008 IBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚60 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚154610182094848 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 1661528824832 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚44038144 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚262 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚262 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚177360771448832 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚429 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 1568236919271 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚45886934 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚21820517 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚20441711 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚223 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚5458479 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 845178880 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚322 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚322 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 122306526 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚14983163 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚24066415 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 831258624 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 5368709120 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 5368709120 @7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚5264 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚178932339249152 M@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚59049984 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 1571550695424 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚14983161 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚322 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 KBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚4876 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 _step‚533 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 4552982528 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 1571550695424 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4289 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚61478912 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 4517085184 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚156271646237696 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚177360771448832 IBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚71 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 5494538240 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚322 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 4654602240 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚178932280199168 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚4289 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚4287 PCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚58387380 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 4628521206 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚21816230 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚41945088 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 4552982528 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚20441640 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 1661484786688 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 KDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚60 trainer/global_step‚6499 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚177360788553728 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚176312140201984 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚41348020 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 5368709120 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 125829120 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 125829120 E8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚45882645 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚21820517 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚178932280199168 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 4654602240 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 125829120 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚262 % _timestamp‚1.7866604049582705e+09 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17104896 PCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚59049984 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚177880435508635 F>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚209 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚20643840 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 _runtime‚1753.491899252•Œ¸úw2Ó- _wandbruntime‚1753  _runtime‚1753£$ž‘Ç&ÔÁM S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚177906793487403 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚262 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚177906851874831 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚41348068 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 122306526 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654602240 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚45889485 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚4293 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 831258624 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 4552982528 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 1568467491883 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 5368709120 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 125829120 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚45889485 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 122905600 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚223 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚322 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚61478912 NADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚58387428 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚176338325995520 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚5459392 BDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59053056 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 1571781767168 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚20643840 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521206 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 4517085184 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 1568508839951 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚178958892658688 H@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚665 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚20444681 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 4654602240 NGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚77 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚209 DDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚45893780 B9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚5264 G@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚60 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚45893780 >5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5264 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚41948160 SFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚68873216 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 125829120 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17104896 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚322 _runtime‚1753.655302014 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚21823810 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 1661776474112 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 5368709120 E6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚322 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚176338343034880 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚177387110891520 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 4552982528 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚24069970 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 5368709120 IBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚60 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚21823810 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚21819517 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚177387093786624 NADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚59053056 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 1571739819008 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 1661728244736 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚177387110891520 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚177387093786624 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 PDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚4293 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚21819517 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚154633229467648 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 5494538240 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚154633208823808 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚48229376 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚322 D=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚60 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚24069968 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 % _timestamp‚1.7866604051249065e+09 _step‚534 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚14985368 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚156294937068544 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 1571781767168 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 1571739819008 E5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5264 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚41948160 SFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚68873216 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 125829120 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17104896 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚322 _runtime‚1753.655302014 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚21823810 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 1661776474112 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 5368709120 E6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚322 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚176338343034880 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚177387110891520 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 4552982528 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚24069970 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 5368709120 IBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚60 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚21823810 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚21819517 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚177387093786624 NADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚59053056 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 1571739819008 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 1661728244736 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚177387110891520 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚177387093786624 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 PDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚4293 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚21819517 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚154633229467648 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 5494538240 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚154633208823808 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚48229376 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚322 D=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚60 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚24069968 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 % _timestamp‚1.7866604051249065e+09 _step‚534 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚14985368 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚156294937068544 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 1571781767168 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 1571739819008 EDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654602240 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚45889485 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚4293 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 831258624 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 4552982528 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 1568467491883 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 5368709120 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 125829120 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚45889485 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 122905600 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚223 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚322 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚61478912 NADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚58387428 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚176338325995520 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚5459392 BDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59053056 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 1571781767168 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚20643840 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521206 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 4517085184 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 1568508839951 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚178958892658688 H@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚665 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚20444681 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 4654602240 NGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚77 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚209 DDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚45893780 B9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚5264 G@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚60 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚45893780–!Q¨2Ö- _wandbruntime‚1753  _runtime‚1753ÝÓpšs×í _step‚535 0loss/FRt1/assignment_loss‚0.7377211836105191 /loss/FRt2/detection_loss‚0.5110604189301755 epoch‚16 % _timestamp‚1.7866604051259446e+09 _runtime‚1753.655800856 0loss/FRt2/assignment_loss‚0.6742551303726609 /loss/FRt1/detection_loss‚0.5625450242544399 &loss/total_loss‚2.4855817571677954 trainer/global_step‚6499šm‚ßxØò 0loss/FRt1/assignment_loss‚0.7377211836105191 /loss/FRt2/detection_loss‚0.5110604189301755 epoch‚16 % _timestamp‚1.7866604051259446e+09 _runtime‚1753.655800856 0loss/FRt2/assignment_loss‚0.6742551303726609 /loss/FRt1/detection_loss‚0.5625450242544399 &loss/total_loss‚2.4855817571677954 trainer/global_step‚6499 _step‚535—ÕüÔ»–Ù‚(:‹ ¶ŒùÓ‚þ˜á gpu.0.gpu‚60.0 gpu.0.memory‚11, gpu.0.memoryAllocated‚7.0313262939453125) gpu.0.memoryAllocatedBytes‚ 6039863296 gpu.0.temp‚35.0 gpu.0.powerWatts‚238.073' gpu.0.enforcedPowerLimitWatts‚500.0 gpu.0.powerPercent‚47.6146 gpu.0.smClock‚1410 gpu.0.memoryClock‚1593$ gpu.0.correctedMemoryErrors‚1955# gpu.0.uncorrectedMemoryErrors‚0DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4876 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚262 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 F>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚180259983854592 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 1583137461248 IDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17104896 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚155762530091008 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚41348020 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 4654602240 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚209 C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚4289 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚157436290606080 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚180259983854592 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚178676888338432 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 5368709120 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚21985619 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 D;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚5264 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚223 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 1579798729879 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚178676888338432 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17104896 BDeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4289 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 831258624 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚177621429878784 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 4517085184 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 122306526 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 1583095516160 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚155762550734848 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 125829120 M@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚59049984 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚157436355288064 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚322 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 122905600 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 1673804553216 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚44038144 IBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚60 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 1583137461248 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚322 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚322 _step‚537 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚180260042904576—*– Á'à»O L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚24244065 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 5494538240 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚4287 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 5494538240 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚21981332 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚24244065 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚179201228608663 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 125829120 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 KBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚4876 PCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚58387380 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 5368709120 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 5368709120 F?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚60 IDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17104896 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚155762530091008 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚41348020 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 4654602240 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚209 C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚4289 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚157436290606080 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚180259983854592 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚178676888338432 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 5368709120 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚21985619 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 D;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚5264 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚223 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 1579798729879 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚178676888338432 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17104896 BDeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4289 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 831258624 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚177621429878784 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 4517085184 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 122306526 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 1583095516160 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚155762550734848 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 125829120 M@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚59049984 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚157436355288064 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚322 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 122905600 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 1673804553216 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚44038144 IBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚60 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 1583137461248 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚322 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚322 _step‚537 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚180260042904576 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚180260042904576 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 4552982528 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚41945088 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 122905600 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚21985619 IBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚63 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚262 KDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚60 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚15093413 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 trainer/global_step‚6549 G>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4876 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚262 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 F>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚180259983854592 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 1583137461248 IDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521206 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 122306526 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚21988912 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚178703227781120 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 122905600 B9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚5264 SFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚64678912 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 831258624 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚322 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 1583368532992 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚46232237 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚46236532 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚178703210676224 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚262 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚262 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 1583326584832 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚21984619 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 1583326584832 DDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 5368709120 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 1580070650559 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 5494538240 D=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚60 PDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654602240 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚24247618 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚177647615672320 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 _step‚538 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚429 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚5499791 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚24247618 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚46236532 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 1674004300800 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 G@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚60 >5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5264 EDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚20595407 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚155785598107648 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚322 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚24247620 H@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚665 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚179227703362239 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚322 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚44035072 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 4517085184 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 5368709120 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚4293 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚24247620 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚209 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚179227644974811 trainer/global_step‚6549 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 122905600 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚155785577463808 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚5499859 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚46232237 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚4293 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 4552982528 E8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 125829120 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚20595477 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚177647632711680 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚262 _runtime‚1766.390330837 K>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59053056 G@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚70 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 1580029302491 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚4295 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚41948160 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚21984619 NGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚68lý|#4ãÅM NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚24247618 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚177647615672320 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 _step‚538 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚429 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚5499791 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚24247618 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚46236532 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 1674004300800 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 G@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚60 >5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5264 EDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚20595407 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚155785598107648 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚322 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚24247620 H@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚665 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚179227703362239 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚322 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚44035072 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 4517085184 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 5368709120 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚4293 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚24247620 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚209 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚179227644974811 trainer/global_step‚6549 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 122905600 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚155785577463808 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚5499859 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚46232237 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚4293 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 4552982528 E8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 125829120 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚20595477 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 ^JDeviceStatsMon®ô^—itor.on_train_batch_end/requested_bytes.large_pool.allocated‚177647632711680 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚262 _runtime‚1766.390330837 K>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59053056 G@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚70 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 1580029302491 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚4295 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚41948160 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚21984619 NGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚68 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 5494538240 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 125829120 NADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚59053056 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚15095618 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚15095616 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚223 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 845178880 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17104896 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚20643840 NADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚58387428 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521206 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 122306526 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚21988912 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚178703227781120 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 122905600 B9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚5264 SFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚64678912 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 831258624 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚322 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 1583368532992 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚46232237 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚46236532 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚178703210676224 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚262 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚262 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 1583326584832 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚21984619 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 1583326584832 DDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 5368709120 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 1580070650559 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 5494538240 D=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚60 PDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654602240šîò2ä- _wandbruntime‚1766  _runtime‚1766èƒQ:qåë 0loss/FRt2/assignment_loss‚0.6901218013694775 /loss/FRt2/detection_loss‚0.5230544055970348 %loss/total_loss‚2.486921381905895 trainer/global_step‚6549 $ _timestamp‚1.786660417861098e+09 _runtime‚1766.391305669 0loss/FRt1/assignment_loss‚0.7403215426771795 /loss/FRt1/detection_loss‚0.5334236322622035 epoch‚16 _step‚539Zôàvæð _runtime‚1766.391305669 0loss/FRt1/assignment_loss‚0.7403215426771795 /loss/FRt1/detection_loss‚0.5334236322622035 epoch‚16 _step‚539 0loss/FRt2/assignment_loss‚0.6901218013694775 /loss/FRt2/detection_loss‚0.5230544055970348 %loss/total_loss‚2.486921381905895 trainer/global_step‚6549 $ _timestamp‚1.786660417861098e+09›½¶Ë¡ç‚(:– ÅŒùÓ£åá gpu.0.gpu‚64.0 gpu.0.memory‚12, gpu.0.memoryAllocated‚7.0313262939453125) gpu.0.memoryAllocatedBytes‚ 6039863296 gpu.0.temp‚36.0 gpu.0.powerWatts‚202.334' gpu.0.enforcedPowerLimitWatts‚500.0) gpu.0.powerPercent‚40.466800000000006 gpu.0.smClock‚1410 gpu.0.memoryClock‚1593$ gpu.0.correctedMemoryErrors‚1955# gpu.0.uncorrectedMemoryErrors‚0B0’cšè‚(: ÅŒùÓ·¡çä( disk.nvme0c0n1.out‚53518.29736328125 disk.sda1.in‚0 disk.sda1.out‚0& disk.nvme0n1.out‚53518.29736328125 disk.dm-1.in‚ 16568.21875 disk.drbd2165.in‚0* proc.memory.percent‚14.220786094665527 network.recv‚ 3.506287e+06% disk./.usageGB‚487.52994537353516 disk.sda3.in‚ 16568.21875$ disk.nvme0c0n1.in‚43228.69921875 disk.sda.in‚ 16568.21875 disk.sda2.out‚0 disk.dm-0.in‚43228.69921875$ memory_percent‚66.49138927459717 network.sent‚ 8.739614e+06) disk./.usagePercent‚32.71854369888861 disk.sda2.in‚0 disk.dm-1.out‚ 34.79296875 disk.drbd2165.out‚0) proc.memory.availableMB‚ 5490.05078125# proc.memory.rssMB‚ 2329.93359375 disk.sda3.out‚ 34.79296875 disk.sda.out‚ 34.79296875" disk.nvme0n1.in‚43228.69921875# disk.dm-0.out‚53518.29736328125 cpu‚27.056499755017732 proc.cpu.threads‚17<Î?2é- _wandbruntime‚1778  _runtime‚1778Rþ qÅê¿ % lr-AdamW/pg1‚0.001159251920578947 % lr-AdamW/pg2‚0.001159251920578947 trainer/global_step‚6599 % _timestamp‚1.7866604301239524e+09 _runtime‚1778.653962417 _step‚540ê" ÊëÄ _step‚540 % lr-AdamW/pg1‚0.001159251920578947 % lr-AdamW/pg2‚0.001159251920578947 trainer/global_step‚6599 % _timestamp‚1.7866604301239524e+09 _runtime‚1778.653962417œ W”Í2ì- _wandbruntime‚1778  _runtime‚1778°PÛP½'í·O NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚209 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 PCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚58387380 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 1591401888507 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 5368709120 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 125829120 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚262 @7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚5264 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 1591360540487 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 5494538240 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚181587687510016 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚179993022332928 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚156914898731008 BDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 4552982528 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 4628521206 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚322 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚4287 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 5368709120 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚223 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚5540089 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 125829120 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚262 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚20743750 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚22150721 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 4517085184 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 1594724227072 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚5540161 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚322 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚322 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚24421715 UHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚73070592 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 831258624 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚61478912 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4289 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 5494538240 trainer/global_step‚6599 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚179993005228032 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 1594682281984 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 845178880 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚181587746560000 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 4654602240 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚24421715 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 5494538240 _step‚541 C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚4289 M@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚59049984 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚179993005228032 JBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚665 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 5368709120 _runtime‚1778.658905195 IDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4876 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚178930736594944 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚15203663 KDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚60 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚41945088 IDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚181587687510016 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 4654602240 PCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚59049984 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 1686035685376 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17104896 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 122905600 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚4287 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚22146434 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚181587746560000 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 KBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚4876 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚179993022332928 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚24421717 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚20643840 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚22146434 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚156914919374848 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 122306526ø§:rÂ'î¼O E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 4628521206 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚322 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚4287 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 5368709120 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚223 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚5540089 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 125829120 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚262 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚20743750 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚22150721 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 4517085184 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 1594724227072 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚5540161 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚322 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚322 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚24421715 UHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚73070592 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 831258624 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚61478912 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4289 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 5494538240 trainer/global_step‚6599 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚179993005228032 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 1594682281984 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 845178880 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚181587746560000 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 4654602240 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚24421715 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 5494538240 _step‚541 C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚4289 M@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚59049984 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚179993005228032 JBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚665 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 5368709120 _runtime‚1778.658905195 IDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4876 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚178930736594944 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚15203663 KDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚60 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚41945088 IDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚181587687510016 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 4654602240 PCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚59049984 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 1686035685376 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17104896 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 122905600 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚4287 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚22146434 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚181587746560000 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 KBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚4876 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚179993022332928 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚24421717 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚20643840 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚22146434 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚156914919374848 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 122306526 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚209 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 PCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚58387380 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 1591401888507 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 5368709120 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 125829120 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚262 @7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚5264 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 1591360540487 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 5494538240 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚181587687510016 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚179993022332928 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚156914898731008 BDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 4552982528 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0Ô ‡2ï- _wandbruntime‚1778  _runtime‚1778C&–èð¿M D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521206 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚262 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚262 # _timestamp‚1.78666043028565e+09 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚5540895 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚158624300051456 SFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚73067520 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 trainer/global_step‚6599 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚46574989 NGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚69 H@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚665 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚5540964 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚209 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚156937966747648 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚24425270 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 4552982528 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 122905600 B9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚5264 I@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚4876 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 1686280880128 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 4517085184 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚46579284 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚24425268 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 1591632461167 G@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚71 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚223 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 4552982528 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚322 EDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654602240 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚41948160 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚46579284 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚22149721 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚20643840 _runtime‚1778.815969793 >5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5264 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚4293 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 1594955298816 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚52423680 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 5368709120 D=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚60 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 831258624 NADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚59053056 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚24425268 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚178956905349120 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool..pDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚46574989 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚181614299969536 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚20746761 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 1594955298816 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 122905600 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚156937946103808 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚61478912 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚322 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚4293 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚158624226983936 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 5494538240 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚322 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚22154014 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚41348068 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 125829120 IBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚60 _step‚542 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚22154014 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚22149721 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚180548554849647 NADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚58387428 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 125829120 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚181614240916480 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚20746832 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 1686333303808 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚322 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 122306526 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚180019327565824 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚180019344670720 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 5368709120 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚262 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 5368709120 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚181614299969536 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚180019344670720 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚4295 EDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59053056=?°dÊ&ñÄM \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 1594955298816 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 122905600 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚156937946103808 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚61478912 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚322 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚4293 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚158624226983936 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 5494538240 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚322 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚22154014 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚41348068 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 125829120 IBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚60 _step‚542 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚22154014 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚22149721 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚180548554849647 NADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚58387428 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 125829120 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚181614240916480 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚20746832 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 1686333303808 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚322 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 122306526 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚180019327565824 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚180019344670720 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 5368709120 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚262 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 5368709120 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚181614299969536 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚180019344670720 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚4295 EDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59053056 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521206 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚262 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚262 # _timestamp‚1.78666043028565e+09 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚5540895 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚158624300051456 SFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚73067520 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 trainer/global_step‚6599 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚46574989 NGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚69 H@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚665 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚5540964 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚209 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚156937966747648 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚24425270 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 4552982528 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 122905600 B9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚5264 I@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚4876 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 1686280880128 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 4517085184 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚46579284 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚24425268 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 1591632461167 G@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚71 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚223 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 4552982528 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚322 EDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654602240 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚41948160 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚46579284 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚22149721 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚20643840 _runtime‚1778.815969793 >5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5264 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚4293 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 1594955298816 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚52423680 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 5368709120 D=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚60 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 831258624 NADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚59053056 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚24425268 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚178956905349120 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 1591591113099 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 5494538240 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚46574989 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚181614299969536 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚20746761 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0ž+þa2ò- _wandbruntime‚1778  _runtime‚1778¬l>oóé .loss/FRt1/assignment_loss‚0.70929920456469 0loss/FRt2/assignment_loss‚0.6395996745913446 /loss/FRt1/detection_loss‚0.5143824120732543 /loss/FRt2/detection_loss‚0.4675312398316199 epoch‚16 % _timestamp‚1.7866604302865405e+09 _step‚543 %loss/total_loss‚2.330812531060909 trainer/global_step‚6599 _runtime‚ 1778.81633325^¦Ü tôî /loss/FRt2/detection_loss‚0.4675312398316199 epoch‚16 % _timestamp‚1.7866604302865405e+09 _step‚543 %loss/total_loss‚2.330812531060909 trainer/global_step‚6599 _runtime‚ 1778.81633325 .loss/FRt1/assignment_loss‚0.70929920456469 0loss/FRt2/assignment_loss‚0.6395996745913446 /loss/FRt1/detection_loss‚0.5143824120732543Ÿ^žfΖõ‚(:‹ ÔŒùÓ³¬ á gpu.0.gpu‚64.0 gpu.0.memory‚12, gpu.0.memoryAllocated‚7.0313262939453125) gpu.0.memoryAllocatedBytes‚ 6039863296 gpu.0.temp‚33.0 gpu.0.powerWatts‚119.454' gpu.0.enforcedPowerLimitWatts‚500.0 gpu.0.powerPercent‚23.8908 gpu.0.smClock‚1410 gpu.0.memoryClock‚1593$ gpu.0.correctedMemoryErrors‚1955# gpu.0.uncorrectedMemoryErrors‚0ãëvÌ•ö‚(:Š ÔŒùÓúàÅä disk.sda.out‚ 34.984375& disk.nvme0n1.out‚53522.51220703125$ memory_percent‚66.87140464782715# proc.memory.rssMB‚ 2329.93359375 proc.cpu.threads‚17 network.sent‚ 8.813099e+06 disk.dm-0.in‚43239.23046875 disk.sda3.in‚ 16589.078125 disk.sda1.in‚0 disk.sda2.out‚0" disk.nvme0n1.in‚43239.23046875( proc.memory.availableMB‚ 5427.7890625) disk./.usagePercent‚32.71812615060026 disk.dm-1.out‚ 34.984375 disk.drbd2165.in‚0 disk.sda1.out‚0 disk.sda2.in‚0* proc.memory.percent‚14.220786094665527 network.recv‚ 3.537575e+06 disk.dm-1.in‚ 16589.078125 disk.drbd2165.out‚0 disk.sda3.out‚ 34.984375$ disk.nvme0c0n1.in‚43239.23046875( disk.nvme0c0n1.out‚53522.51220703125 disk.sda.in‚ 16589.078125 cpu‚27.056727340204322$ disk./.usageGB‚487.5237236022949# disk.dm-0.out‚53522.51220703125Ùåyr2÷- _wandbruntime‚1791  _runtime‚1791Å$"Åø¿ trainer/global_step‚6649 % _timestamp‚1.7866604429061646e+09 _runtime‚1791.436156529 _step‚544 % lr-AdamW/pg1‚0.001154201861038138 % lr-AdamW/pg2‚0.001154201861038138[›ÊùÄ _runtime‚1791.436156529 _step‚544 % lr-AdamW/pg1‚0.001154201861038138 % lr-AdamW/pg2‚0.001154201861038138 trainer/global_step‚6649 % _timestamp‚1.7866604429061646e+09 [5 2ú- _wandbruntime‚1791  _runtime‚1791âŸä†½'û·O @7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚5264 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚24599365 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4289 D;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚5264 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚22315823 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 125829120 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 1606269047808 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 1698314801664 IBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚60 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚4287 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚180240009232384 _step‚545 PCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚58387380 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 M@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚59049984 C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚4289 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚182915391165440 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚4287 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 4552982528 IDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 1602922351095 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚24599367 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚22311536 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚181309139222528 KBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚4876 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚429 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 831258624 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚15313911 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 5494538240 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 125829120 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17104896 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚46915190 JBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚665 PIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚65 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 5368709120 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 1606269047808 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 5368709120 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 % _timestamp‚1.7866604429070764e+09 PCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚59049984 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚22311536 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚46135296 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚181842931583479 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 5368709120 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚322 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚41945088 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚209 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚322 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚262 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚182915450215424 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 1606310992896 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚5581301 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 125829120 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚262 E8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚46910901 KDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚60 IDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4876 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 4552982528 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 5494538240 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚22315823 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚20895279 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚20895212 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚20643840 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 F?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚60 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚181309122117632 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚181309122117632 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 1606310992896 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17104896 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 122905600 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚223 UHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚66779136 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 4517085184 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 4654602240 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚24599365 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚5581366 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 4628521206 _runtime‚1791.440899869 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 122905600 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 845178880 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚158067288014848 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚158067267371008 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚322 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 5494538240 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 BDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚182915391165440 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚4287 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 4552982528 IDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 1602922351095 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚24599367 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚22311536 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚181309139222528 KBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚4876 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚429 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 831258624 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚15313911 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 5494538240 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 125829120 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17104896 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚46915190 JBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚665 PIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚65 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 5368709120 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 1606269047808 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 5368709120 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 % _timestamp‚1.7866604429070764e+09 PCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚59049984 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚22311536 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚46135296 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚181842931583479 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 5368709120 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚322 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚41945088 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚209 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚322 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚262 VBDeviceStatsMonitor.ëKon_train_batch_start/active_bytes.all.allocated‚182915450215424 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 1606310992896 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚5581301 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 125829120 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚262 E8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚46910901 KDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚60 IDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4876 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 4552982528 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 5494538240 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚22315823 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚20895279 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚20895212 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚20643840 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 F?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚60 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚181309122117632 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚181309122117632 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 1606310992896 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17104896 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 122905600 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚223 UHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚66779136 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 4517085184 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 4654602240 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚24599365 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚5581366 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 4628521206 _runtime‚1791.440899869 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 122905600 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 845178880 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚158067288014848 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚158067267371008 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚322 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 5494538240 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 BDeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4289 D;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚5264 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚22315823 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 125829120 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 1606269047808 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 1698314801664 IBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚60 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚4287 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚180240009232384 _step‚545 PCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚58387380 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0¡ Ð_2ý- _wandbruntime‚1791  _runtime‚1791“Q1Ç&þÁM P6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚322 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 % _timestamp‚1.7866604430831656e+09 >5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5264 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚20643840 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 1698557374976 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521206 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚322 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17104896 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚182941944571904 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 5494538240 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 EDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚223 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚5582114 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 1603194271775 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 5494538240 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚262 _runtime‚1791.613506894 K>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59053056 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 122905600 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 5368709120 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚262 E6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚322 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚20898230 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚181869406337055 IBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚60 DDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 4552982528 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚41348068 BDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654602240 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚24602918 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚41948160 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚24602918 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 1603152923707 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚46922036 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 4517085184 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚41948160 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 1606500116480 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚15316118 NGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚64 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 845178880 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚15316116 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚4295 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚4293 NADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚59053056 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚61478912 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚180266212065280 H@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚665 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚22319116 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 1606542064640 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚20898296 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 125829120 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚181335461560320 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚4293 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚159788938894848 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 NADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚58387428 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 122905600 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚181869347949627 _step‚546‚ ûÌ&ÿÆM ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚180266195025920 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚182942003624960 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚46922036 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 4654602240 I@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚4876 G@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚66 G@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚60 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 1606542064640 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚22319116 SFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚66776064 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 1698603507200 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚262 B9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚5264 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚22314823 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 1606500116480 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚429 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚158090335387648 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚159788872118784 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 trainer/global_step‚6649 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚46917741 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17104896 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚5582178 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 125829120 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚322 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚322 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚20898230 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚181869406337055 IBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚60 DDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 4552982528 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚41348068 BDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654602240 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚24602918 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚41948160 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚24602918 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 1603152923707 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚46922036 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 4517085184 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚41948160 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 1606500116480 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚15316118 NGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚64 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 845178880 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚15316116 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚4295 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚4293 NADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚59053056 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚61478912 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚180266212065280 H@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚665 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚22319116 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 1606542064640 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚20898296 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 125829120 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚181335461560320 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚4293 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚159788938894848 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 NADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚58387428 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 122905600 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚181869347949627 _step‚546 P6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚322 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 % _timestamp‚1.7866604430831656e+09 >5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5264 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚20643840 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 1698557374976 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521206 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚322 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17104896 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚182941944571904 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 5494538240 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 EDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚223 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚5582114 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 1603194271775 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 5494538240 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚262 _runtime‚1791.613506894 K>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59053056 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 122905600 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 5368709120 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚262 EDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚262 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 1617897758720 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 1617855813632 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚181549298909184 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚182625239007232 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚160930208436736 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚159219636011008 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 1614484161703 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚322 C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚4289 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚184243153870848 PCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚59049984 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚21046256 PCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚58387380 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚262 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚21046316 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 5368709120 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚262 KBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚4876 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚184243094820864 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 1617855813632 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚223 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚5622153 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 F>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚22480925 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚182625256112128 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚41945088 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 4654602240 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚15424163 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 4654602240 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4289 PIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚58 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚20643840 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 5494538240 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 5494538240 @8DeviceStatsMonitor.on_train_batch_start/segÄàŠf®ment.all.peak‚322 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚24777015 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚22480925 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 _runtime‚1804.141883684 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 122905600 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚4287 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚41940992 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚184243094820864 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚159219656654848 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 122306526 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚183163783070887 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 125829120 IDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4876 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚47257942 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚429 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 E8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚47253653 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 122905600 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚182625256112128 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17104896 IBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚60 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚160930271021568 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚41348020 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 831258624 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚5622095 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 4628521206 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 4517085184 KDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚60 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 1617897758720 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 4552982528 IDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚22480925 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚182625256112128 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚41945088 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 4654602240 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚15424163 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 4654602240 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4289 PIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚58 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚20643840 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 5494538240 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 5494538240 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚322 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚24777015 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚22480925 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 _runtime‚1804.141883684 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 122905600 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚4287 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚41940992 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚184243094820864 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚159219656654848 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 122306526 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚183163783070887 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 125829120 IDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4876 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚47257942 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚429 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 E8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚47253653 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 122905600 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚182625256112128 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17104896 IBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚60 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚160930271021568 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚41348020 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 831258624 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚5622095 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 4628521206 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 4517085184 KDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚60 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 1617897758720 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 4552982528 IDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚262 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 1617897758720 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 1617855813632 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚181549298909184 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚182625239007232 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚160930208436736 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚159219636011008 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 1614484161703 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚322 C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚4289 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚184243153870848 PCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚59049984 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚21046256 PCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚58387380 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚262 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚21046316 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 5368709120 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚262 KBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚4876 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚184243094820864 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 1617855813632 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚223¥)Í$2‹- _wandbruntime‚1804  _runtime‚1804CÑRÆ&ŒÀM D=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚60 NADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚59053056 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚181575484702720 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 1614714734315 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 5368709120 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚262 _runtime‚1804.301980195 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚184269707280384 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚22479925 BDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚22479925 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚21049321 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚429 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚15426366 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚47260493 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 4654602240 B9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚5264 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚21049250 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚159242704027648 NADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚58387428 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚47264788 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚184269648227328 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚182651578449920 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚24780570 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚159242683383808 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚22484218 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 4552982528 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 1618128830464 I@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚4876 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 845178880 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 1710813590016 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 5494538240 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚183190199437035 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚181575501742080 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 4517085184 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚24780570 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚4293 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 122306526 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚262 K>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59053056 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚182651578449920 >5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5264 NGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚69 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 831258624 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚262 _step‚550 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚223 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚322 trainer/global_step‚6699 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 1618128830464 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 SFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚66776064 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 125829120 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 4552982528 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚160953496973824 D6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚322 G@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚60 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚41948160 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚160953563749888 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚183190257824463 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚182651561345024 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 122905600 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 1710859722240 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521206 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 5368709120 PDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654602240 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚5622884 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚46132224 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚61478912 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 IBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚60º›ß`uÅM E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚21049250 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚159242704027648 NADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚58387428 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚47264788 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚184269648227328 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚182651578449920 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚24780570 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚159242683383808 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚22484218 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 4552982528 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 1618128830464 I@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚4876 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 845178880 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 1710813590016 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 5494538240 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚183190199437035 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚181575501742080 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 4517085184 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚24780570 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚4293 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 122306526 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚262 K>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59053056 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚182651578449920 >5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5264 NGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚69 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 831258624 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚262 _step‚550 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚223 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚322 trainer/global_step‚6699 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 1618128830464 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 SFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚66776064 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 125829120 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 4552982528 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚160953496973824 D6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚322 G@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚60 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚41948160 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚160953563749888 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚183190257824463 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚182651561345024 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 122905600 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 1710859722240 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521206 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 5368709120 PDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654602240 QEDeviceStatsMonitor.on_train_batch_end/inactiv G’V e_split.small_pool.freed‚5622884 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚46132224 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚61478912 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 IBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚60 D=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚60 NADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚59053056 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚181575484702720 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 1614714734315 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 5368709120 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚262 _runtime‚1804.301980195 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚184269707280384 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚22479925 BDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚22479925 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚21049321 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚429 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚15426366 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚47260493 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 4654602240 B9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚5264¦µfû2Ž- _wandbruntime‚1804  _runtime‚1804ÈÍÅqë %loss/total_loss‚2.631791701826966 epoch‚16 trainer/global_step‚6699 % _timestamp‚1.7866604557726445e+09 _runtime‚1804.302444409 /loss/FRt1/assignment_loss‚0.830818858273455 /loss/FRt1/detection_loss‚0.5299393414639795 /loss/FRt2/detection_loss‚0.5114731019377136 _step‚551 0loss/FRt2/assignment_loss‚0.75956040015181790JR“vð 0loss/FRt2/assignment_loss‚0.7595604001518179 %loss/total_loss‚2.631791701826966 epoch‚16 trainer/global_step‚6699 % _timestamp‚1.7866604557726445e+09 _runtime‚1804.302444409 /loss/FRt1/assignment_loss‚0.830818858273455 /loss/FRt1/detection_loss‚0.5299393414639795 /loss/FRt2/detection_loss‚0.5114731019377136 _step‚551§üçyk”‘‚(:‰ òŒùÓÕû¯á gpu.0.gpu‚0.0 gpu.0.memory‚0, gpu.0.memoryAllocated‚7.0313262939453125) gpu.0.memoryAllocatedBytes‚ 6039863296 gpu.0.temp‚33.0 gpu.0.powerWatts‚119.454' gpu.0.enforcedPowerLimitWatts‚500.0 gpu.0.powerPercent‚23.8908 gpu.0.smClock‚1410 gpu.0.memoryClock‚1593$ gpu.0.correctedMemoryErrors‚1955# gpu.0.uncorrectedMemoryErrors‚0Q‰šä ’‚(:• òŒùÓ¡‰†ã disk.sda2.out‚0 network.sent‚ 8.944516e+06 disk.drbd2165.in‚0 disk.drbd2165.out‚0 disk.sda1.out‚0 disk.sda2.in‚0" disk.nvme0n1.in‚43260.73828125% disk.nvme0n1.out‚53574.0380859375$ memory_percent‚66.87407493591309 network.recv‚ 3.582375e+06 disk.dm-1.out‚ 35.55859375 disk.sda3.out‚ 35.55859375# proc.memory.rssMB‚ 2329.93359375* proc.memory.percent‚14.220786094665527 cpu‚27.057878903457876* disk./.usagePercent‚32.718121542464644% disk./.usageGB‚487.52365493774414 disk.sda.in‚17790.28515625 disk.sda1.in‚0( proc.memory.availableMB‚ 5427.3515625 proc.cpu.threads‚17' disk.nvme0c0n1.out‚53574.0380859375 disk.dm-1.in‚17790.28515625 disk.dm-0.in‚43260.73828125" disk.dm-0.out‚53574.0380859375 disk.sda3.in‚17790.28515625$ disk.nvme0c0n1.in‚43260.73828125 disk.sda.out‚ 35.558593755•”¿2“- _wandbruntime‚1816  _runtime‚1816ÛCÇ”Á _step‚552 & lr-AdamW/pg1‚0.0011440241845891775 & lr-AdamW/pg2‚0.0011440241845891775 trainer/global_step‚6749 % _timestamp‚1.7866604681109962e+09 _runtime‚1816.641001395 ÊÈÌ•Æ & lr-AdamW/pg1‚0.0011440241845891775 & lr-AdamW/pg2‚0.0011440241845891775 trainer/global_step‚6749 % _timestamp‚1.7866604681109962e+09 _runtime‚1816.641001395 _step‚552¨© ?`2–- _wandbruntime‚1816  _runtime‚1816~¤°&¼'—¶O E8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚47596405 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 1629442579456 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚15534413 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 4628521206 _runtime‚1816.645898936 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 1626045972311 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚262 trainer/global_step‚6749 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4289 PIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚79 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 1722896148992 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17104896 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 1629484524544 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚21197148 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚20643840 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚160372004651008 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 4654602240 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 4552982528 D;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚5264 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚24954667 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚184484634558295 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 5494538240 F?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚60 F>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚15534411 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 5368709120 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚322 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚262 IBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚60 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 1629442579456 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 1722847916544 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚24954667 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚4287 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚185570857526272 BDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚185570798476288 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚41945088 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 122905600 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚47600694 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚162094852567552 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 5494538240 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚24954665 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 125829120 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚262 M@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚59049984 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚183941355896832 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚61478912 G>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4876 IDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 _step‚553 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚183941355896832 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 1629484524544 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 845178880 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 831258624 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚183941373001728 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚48232448 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 1626087320331 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 122905600 PCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚59049984 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚5662656 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 5494538240 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚41945088 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚24954665 IBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚81 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 125829120 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚22646027 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚182858605625344 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 5368709120 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚322 $ _timestamp‚1.786660468111886e+09PZq™Á'˜»O E8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚47596405 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 1629442579456 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚15534413 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 4628521206 _runtime‚1816.645898936 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 1626045972311 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚262 trainer/global_step‚6749 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4289 PIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚79 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 1722896148992 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17104896 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 1629484524544 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚21197148 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚20643840 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚160372004651008 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 4654602240 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 4552982528 D;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚5264 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚24954667 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚184484634558295 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 5494538240 F?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚60 F>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚15534411 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 5368709120 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚322 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚262 IBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚60 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 1629442579456 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 1722847916544 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚24954667 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚4287 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚185570857526272 BDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚185570798476288 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚41945088 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 122905600 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚47600694 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚162094852567552 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 5494538240 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚24954665 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 125829120 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚262 M@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚59049984 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚183941355896832 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚61478912 G>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4876 IDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 _step‚553 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚183941355896832 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 1629484524544 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 845178880 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 831258624 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚183941373001728 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚48232448 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 1626087320331 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 122905600 PCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚59049984 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚5662656 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 5494538240 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚41945088 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚24954665 IBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚81 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 125829120 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚22646027 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚182858605625344 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 5368709120 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚322 $ _timestamp‚1.786660468111886e+09©L ²Û2™- _wandbruntime‚1816  _runtime‚1816Ël‡UšÁM J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚22645027 K>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59053056 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚184511050924443 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 5368709120 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚262 EDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚21200076 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 4654602240 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521206 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚160395072667648 NADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚58387428 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 PDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654602240 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 1723093275136 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 B8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 5494538240 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚24958220 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 122905600 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚183967678234624 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚24958218 SFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚70970368 D5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5264 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 122905600 G@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚60 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚162118216269312 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚50326528 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 5368709120 % _timestamp‚1.7866604682739096e+09 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚4293 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17104896 NADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚59053056 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17104896 D=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚60 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚47603245 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 5494538240 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 125829120 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚262 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚47607540 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚429 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 831258624 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚322 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 125829120 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚322 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 G@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚79 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚5663537Ð/ Ì&›ÆM NGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚77 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 1626276544923 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 1629673648128 H@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚665 I@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚4876 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚21200155 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 1629673648128 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚22645027 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚15536616 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚20643840 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 1626317892991 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚41348068 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654602240 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 1723093275136 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 B8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 5494538240 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚24958220 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 122905600 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚183967678234624 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚24958218 SFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚70970368 D5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5264 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 122905600 G@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚60 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚162118216269312 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚50326528 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 5368709120 % _timestamp‚1.7866604682739096e+09 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚4293 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17104896 NADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚59053056 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17104896 D=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚60 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚47603245 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 5494538240 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 125829120 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚262 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚47607540 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚429 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 831258624 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚322 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 125829120 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚322 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 G@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚79 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚5663537 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚22645027 K>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59053056 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚184511050924443 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 5368709120 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚262 EDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚21200076 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 4654602240 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521206 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚160395072667648 NADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚58387428 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 PDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚41348020 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 5368709120 @7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚5264 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 KBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚4876 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 5494538240 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚262 M@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚59049984 E8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚47939157 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚186898561181696 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚322 trainer/global_step‚6799 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚15644661 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚22806842 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚185257489891328 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚184167895302144 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 5494538240 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚25132317 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚20643840 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚161524373291008 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 831258624 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 125829120 F>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17104896 IDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚186898502131712 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚185257489891328 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 4552982528 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚161524393934848 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚186898561181696 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚44038144 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 4628521206 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 4654602240 PIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚66 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚322 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 4654602240 D;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚5264 UHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚64681984 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚41945088 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 1637649130939 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚185257472786432 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚185257472786432 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 122905600 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 4552982528 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚4287 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚15644663 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚5702939 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 1637607782919 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚163259557718016 BDeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4289 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚25132315 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚185805486045703 _step‚557 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚25132315 PCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚59049984 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚185805544433083 G>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4876 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 122905600 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚47943446 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚163259493036032 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚4289 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚223 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 1735163783168 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 125829120 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚262 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17104896 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚186898502131712 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚209 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 5494538240 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 1641071290368 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚21347600 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 1735119745024 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚322 F?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚60 JBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚665 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚5703005 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 4517085184 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 5368709120»0[f¨ ¤¼O YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚61478912 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚41348020 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 5368709120 @7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚5264 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 KBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚4876 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 5494538240 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚262 M@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚59049984 E8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚47939157 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚186898561181696 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚322 trainer/global_step‚6799 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚15644661 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚22806842 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚185257489891328 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚184167895302144 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 5494538240 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚25132317 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚20643840 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚161524373291008 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 831258624 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 125829120 F>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17104896 IDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚186898502131712 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚185257489891328 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 4552982528 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚161524393934848 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚186898561181696 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚44038144 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 4628521206 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 4654602240 PIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚66 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚322 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 4654602240 D;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚5264 UHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚64681984 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚41945088 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 1637649130939 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚185257472786432 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚185257472786432 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 122905600 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 4552982528 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚4287 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚15644663 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚5702939 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 1637607782919 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚163259557718016 BDeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4289 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚25132315 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚185805486045703 _step‚557 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚25132315 PCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚59049984 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚185805544433083 G>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4876 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 122905600 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚47943446 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚163259493036032 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚4289 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚223 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.ÔÜ-~small_pool.allocated‚ 1735163783168 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 125829120 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚262 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17104896 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚186898502131712 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚209 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 5494538240 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 1641071290368 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚21347600 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 1735119745024 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚322 F?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚60 JBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚665 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚5703005 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 4517085184 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 5368709120 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚22811129 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 845178880 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 5368709120 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚25132317 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0­‹¥ä2¥- _wandbruntime‚1829  _runtime‚1829a,HÅ&¦¿M XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 1641260413952 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚161547441307648 NADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚58387428 _runtime‚ 1829.32006975 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚186925114591232 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 G@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚75 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚322 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚185283812229120 NADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚59053056 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 1637879703599 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 5494538240 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 5494538240 D=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚60 trainer/global_step‚6799 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚186925055538176 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 1637838355531 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17104896 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 EDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654602240 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚322 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚22810129 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 4552982528 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚41948160 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚209 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚322 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚185831960799279 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚262 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚25135868 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚5703833 SFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚64678912 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚25135868 I@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚4876 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚44035072 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 5368709120 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚4293 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚184194064056320 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 125829120 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚47945997 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚4293 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚429 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚15646868 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚163282850806784 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 5494538240 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚186925114591232 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚21350626 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚163282786127872 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 845178880 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 D6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚322 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 1641260413952 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 4552982528 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 125829120 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 4654602240 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚21350701 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚161547420663808 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 NGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚73 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚41348068 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚262 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17104896 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚47945997 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚185283795124224 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521206 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚262 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 122905600 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚41948160 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚61478912 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚4295 >5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5264 K>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59053056 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚185283812229120 H@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚665 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚5703760 G@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚60 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚47950292 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 125829120 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚22810129 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 4517085184 IBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚60 $ _timestamp‚1.786660480789578e+09 _step‚558 P8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚184194064056320 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 125829120 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚47945997 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚4293 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚429 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚15646868 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚163282850806784 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 5494538240 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚186925114591232 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚21350626 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚163282786127872 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 845178880 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 D6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚322 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 1641260413952 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 4552982528 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 125829120 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 4654602240 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚21350701 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚161547420663808 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 NGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚73 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚41348068 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚262 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17104896 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚47945997 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚185283795124224 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521206 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚262 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 122905600 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚41948160 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚61478912 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚4295 >5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5264 K>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59053056 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚185283812229120 H@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚665 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚5703760 G@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚60 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚47950292 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 125829120 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚22810129 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 4517085184 IBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚60 $ _timestamp‚1.786660480789578e+09 _step‚558 PDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚322 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚185283812229120 NADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚59053056 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 1637879703599 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 5494538240 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 5494538240 D=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚60 trainer/global_step‚6799 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚186925055538176 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 1637838355531 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17104896 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 EDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654602240 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚322 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚22810129 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 4552982528 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚41948160 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚209 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚322 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚185831960799279 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚262® QHš2¨- _wandbruntime‚1829  _runtime‚1829&hDq©ë %loss/total_loss‚2.771105734141084 % _timestamp‚1.7866604807905986e+09 _step‚559 0loss/FRt1/assignment_loss‚0.8812495802645829 0loss/FRt2/assignment_loss‚0.7673641844371839 /loss/FRt1/detection_loss‚0.5995160769149532 .loss/FRt2/detection_loss‚0.522975892524364 epoch‚16 trainer/global_step‚6799 _runtime‚1829.320693178û#Œvªð 0loss/FRt1/assignment_loss‚0.8812495802645829 0loss/FRt2/assignment_loss‚0.7673641844371839 /loss/FRt1/detection_loss‚0.5995160769149532 .loss/FRt2/detection_loss‚0.522975892524364 epoch‚16 trainer/global_step‚6799 _runtime‚1829.320693178 %loss/total_loss‚2.771105734141084 % _timestamp‚1.7866604807905986e+09 _step‚559¯Lž”l”«‚(:‰ ùÓáÆ°á gpu.0.gpu‚4.0 gpu.0.memory‚0, gpu.0.memoryAllocated‚7.0313262939453125) gpu.0.memoryAllocatedBytes‚ 6039863296 gpu.0.temp‚33.0 gpu.0.powerWatts‚217.958' gpu.0.enforcedPowerLimitWatts‚500.0 gpu.0.powerPercent‚43.5916 gpu.0.smClock‚1410 gpu.0.memoryClock‚1593$ gpu.0.correctedMemoryErrors‚1955# gpu.0.uncorrectedMemoryErrors‚0Ô\¿a’¬‚(:‡ ùÓ׊àä disk.drbd2165.in‚0 disk.drbd2165.out‚0 disk.dm-0.in‚43260.73828125) disk./.usagePercent‚32.73491077256766 disk.sda.in‚17790.28515625 disk.sda1.in‚0 disk.sda2.in‚0 disk.sda2.out‚0% disk.nvme0n1.out‚53620.8408203125" disk.dm-0.out‚53620.8408203125$ memory_percent‚66.87285900115967 network.sent‚ 9.012551e+06 disk.sda3.out‚35.8125$ disk.nvme0c0n1.in‚43260.73828125) proc.memory.availableMB‚ 5427.55078125* proc.memory.percent‚14.220786094665527 cpu‚27.06017458657806$ disk./.usageGB‚487.7738265991211 disk.sda3.in‚17790.28515625 disk.sda.out‚35.8125 disk.sda1.out‚0# proc.memory.rssMB‚ 2329.93359375 proc.cpu.threads‚17 network.recv‚ 3.607773e+06' disk.nvme0c0n1.out‚53620.8408203125" disk.nvme0n1.in‚43260.73828125 disk.dm-1.in‚17790.28515625 disk.dm-1.out‚35.8125Z 2­- _wandbruntime‚1834  _runtime‚1834'Â١߮٠Purity/0FRt/event_purity‚1 2Purity/2FRt/event_proportion‚0.040062315762043 particle/accuracy_1_of_1‚1 epoch‚16 0Purity/1FRt/event_purity‚0.46256184577941895 3Purity/1FRt/event_proportion‚0.6297432780265808 *jet/accuracy_1_of_2‚0.5756350755691528 *loss/val_total_loss‚2.5003148689391246 3Purity/0FRt/event_proportion‚0.3301943838596344 trainer/global_step‚6799 _step‚560 .particle/accuracy_1_of_2‚0.924181342124939 7validation_average_jet_accuracy‚0.31179192662239075 -Purity/2FRt/FRt_purity‚0.5550386905670166 (particle/accuracy‚0.6302781105041504 _runtime‚1834.462788517 % _timestamp‚1.7866604859326735e+09 .Purity/1FRt/FRt_purity‚0.46256184577941895 *jet/accuracy_1_of_1‚0.3066367506980896 /particle/accuracy_2_of_2‚0.6967393755912781 +particle/sensitivity‚0.6480700373649597 +particle/specificity‚0.6205069422721863 /Purity/*FRt/event_purity‚0.4595623016357422 2Purity/*FRt/event_proportion‚0.669805645942688 /Purity/2FRt/event_purity‚0.4107874929904938 'particle/f_score‚0.5543900728225708 +validation_accuracy‚0.20850570499897003 .Purity/*FRt/FRt_purity‚0.47314807772636414 +jet/accuracy_2_of_2‚0.20850570499897003…5rªä¯Þ 7validation_average_jet_accuracy‚0.31179192662239075 -Purity/2FRt/FRt_purity‚0.5550386905670166 (particle/accuracy‚0.6302781105041504 _runtime‚1834.462788517 % _timestamp‚1.7866604859326735e+09 .Purity/1FRt/FRt_purity‚0.46256184577941895 *jet/accuracy_1_of_1‚0.3066367506980896 /particle/accuracy_2_of_2‚0.6967393755912781 +particle/sensitivity‚0.6480700373649597 +particle/specificity‚0.6205069422721863 /Purity/*FRt/event_purity‚0.4595623016357422 2Purity/*FRt/event_proportion‚0.669805645942688 /Purity/2FRt/event_purity‚0.4107874929904938 'particle/f_score‚0.5543900728225708 +validation_accuracy‚0.20850570499897003 .Purity/*FRt/FRt_purity‚0.47314807772636414 +jet/accuracy_2_of_2‚0.20850570499897003 Purity/0FRt/event_purity‚1 2Purity/2FRt/event_proportion‚0.040062315762043 particle/accuracy_1_of_1‚1 epoch‚16 0Purity/1FRt/event_purity‚0.46256184577941895 3Purity/1FRt/event_proportion‚0.6297432780265808 *jet/accuracy_1_of_2‚0.5756350755691528 *loss/val_total_loss‚2.5003148689391246 3Purity/0FRt/event_proportion‚0.3301943838596344 trainer/global_step‚6799 _step‚560 .particle/accuracy_1_of_2‚0.924181342124939°óW’W”°‚(:‰ ùÓé’â gpu.0.gpu‚19.0 gpu.0.memory‚4, gpu.0.memoryAllocated‚7.0313262939453125) gpu.0.memoryAllocatedBytes‚ 6039863296 gpu.0.temp‚32.0 gpu.0.powerWatts‚119.305' gpu.0.enforcedPowerLimitWatts‚500.0 gpu.0.powerPercent‚23.861 gpu.0.smClock‚1410 gpu.0.memoryClock‚1593$ gpu.0.correctedMemoryErrors‚1955# gpu.0.uncorrectedMemoryErrors‚0}L’û—±‚(:Œ ùÓÖçÊä disk.sda1.out‚0( proc.memory.availableMB‚ 5436.1328125# proc.memory.rssMB‚ 2321.36328125 network.recv‚ 3.633024e+06$ memory_percent‚66.82047843933105 cpu‚27.074444266381157 proc.cpu.threads‚17 network.sent‚ 9.100939e+06 disk.nvme0n1.in‚ 43260.84375& disk.nvme0n1.out‚53655.66259765625 disk.drbd2165.out‚0# disk.dm-0.out‚53655.66259765625 disk.sda2.in‚0 disk.sda2.out‚0* proc.memory.percent‚14.168477058410645 disk.dm-1.in‚17790.28515625 disk.dm-1.out‚ 35.9453125 disk.drbd2165.in‚0 disk.sda3.in‚17790.28515625 disk.sda3.out‚ 35.9453125! disk.nvme0c0n1.in‚ 43260.84375( disk.nvme0c0n1.out‚53655.66259765625 disk.sda.out‚ 35.9453125* disk./.usagePercent‚32.734792753094375% disk./.usageGB‚487.77206802368164 disk.dm-0.in‚ 43260.84375 disk.sda.in‚17790.28515625 disk.sda1.in‚0-|ö82²- _wandbruntime‚1847  _runtime‚1847V™!›Æ³À & lr-AdamW/pg2‚0.0011337452800173392 trainer/global_step‚6849 $ _timestamp‚1.786660499224036e+09 _runtime‚1847.754089917 _step‚561 & lr-AdamW/pg1‚0.0011337452800173392Q xË´Å & lr-AdamW/pg2‚0.0011337452800173392 trainer/global_step‚6849 $ _timestamp‚1.786660499224036e+09 _runtime‚1847.754089917 _step‚561 & lr-AdamW/pg1‚0.0011337452800173392±Ö½œ"2µ- _wandbruntime‚1847  _runtime‚1847>áSnL¶·O GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 KDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚60 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 4654602240 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 125829120 C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚4289 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚164727483463680 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚61478912 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 5368709120 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚48368996 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚22998423 @7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚5264 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17104896 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 trainer/global_step‚6849 M@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚59049984 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚188621210728448 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17104896 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 5494538240 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 845178880 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚185858084470784 IDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4876 KBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚4876 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚209 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 125829120 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 PCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚58387380 _runtime‚1847.758902381 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚15795527 E8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚48364707 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚25370573 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 4654602240 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 125829120 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚322 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚262 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚186964543995904 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚22994136 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚54523904 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚185858067431424 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚322 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 122905600 D;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚5264 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚25370573¥(šXq YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚186964526891008 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚41945088 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4289 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚15795525 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚25370571 BDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 UHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚75167744 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚162976012238848 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 4517085184 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 5368709120 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚262 _step‚562 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚186964543995904 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚22994136 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 4628521206 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 1653235565633 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 122306526 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚4287 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 IBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚71 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚429 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 1656683837440 PCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚59049984 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚164727558631424 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 1751471224832 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 5494538240 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 5368709120 PIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚69 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚187511302997057 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚25370571 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚322 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 1656683837440 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚5752611 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚162976032882688 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚41348020 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚21548136 F>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 831258624 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 4552982528 JBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚665 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 1751525748736 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚187511361384437 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚188621210728448 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚4287¾JWçÂ'·¼O UHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚75167744 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚162976012238848 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 4517085184 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 5368709120 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚262 _step‚562 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚186964543995904 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚22994136 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 4628521206 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 1653235565633 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 122306526 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚4287 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 IBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚71 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚429 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 1656683837440 PCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚59049984 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚164727558631424 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 1751471224832 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 5494538240 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 5368709120 PIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚69 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚187511302997057 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚25370571 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚322 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 1656683837440 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚5752611 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚162976032882688 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚41348020 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚21548136 F>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 831258624 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 4552982528 JBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚665 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 1751525748736 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚187511361384437 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚188621210728448 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚4287 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 KDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚60 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 4654602240 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 125829120 C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚4289 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚164727483463680 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚61478912 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 5368709120 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚48368996 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚22998423 @7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚5264 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17104896 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 trainer/global_step‚6849 M@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚59049984 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚188621210728448 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17104896 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 5494538240 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 845178880 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚185858084470784 IDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4876 KBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚4876 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚209 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 125829120 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 PCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚58387380 _runtime‚1847.758902381 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚15795527 E8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚48364707 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚25370573 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 4654602240 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 125829120 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚322 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚262 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚186964543995904 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚22994136 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚54523904 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚185858067431424 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚322 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 122905600 D;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚5264 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚25370573 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚186964526891008 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚41945088 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4289 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚15795525 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚25370571 BDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0²TIÐ\2¸- _wandbruntime‚1847  _runtime‚1847©†©GÆ&¹ÀM J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 4654602240 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚41948160 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚25374126 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚162999080255488 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚185884253224960 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 4517085184 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚186990849228800 I@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚4876 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚5753402 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 5368709120 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚262 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 1656956854272 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚15797732 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚262 IBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚60 G@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚60 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚186990849228800 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚22997423 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚322 _step‚563 B9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚5264 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚164750851556352 BDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚21551132 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚162999059611648 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚48371547 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚61478912 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 125829120 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚4295 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚209 SFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚77261824 NADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚58387428 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚23001716 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚186990866333696 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚322 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 122905600 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚164750774294528 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚187537719363205 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚188647823187968 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚48375842 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚25374126 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚186990866333696 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚322 EDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚48375842 EDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521206 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 $ _timestamp‚1.786660499395006e+09 NGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚82 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚187537777750633 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 1653507486313 trainer/global_step‚6849 K>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59053056 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 1653466138245 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚22997423 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚25374124 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 5494538240 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 D=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚60 H@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚665 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚20643840 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚185884270264320 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚25374124 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚56617984 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚188647823187968 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚5753484 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 1751771300864 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 1751714682880 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚4293 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 4552982528 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 4552982528 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚15797730 >5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5264 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17104896 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 1656914906112 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654602240 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚48371547 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 PDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚41948160 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 122306526 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17104896 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 5494538240 _runtime‚1847.925413848ÎÉž²ºÅM ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚185884270264320 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚25374124 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚56617984 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚188647823187968 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚5753484 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 1751771300864 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 1751714682880 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚4293 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 4552982528 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 4552982528 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚15797730 >5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5264 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17104896 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 1656914906112 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654602240 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚48371547 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 PDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚41948160 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 122306526 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17104896 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 5494538240 _runtime‚1847.925413848 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 4654602240 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚41948160 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚25374126 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚162999080255488 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚185884253224960 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 4517085184 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚186990849228800 I@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚4876 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚5753402 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 5368709120 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚262 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 1656956854272 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚15797732 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚262 IBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚60 G@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚60 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚186990849228800 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚22997423 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚322 _step‚563 B9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚5264 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚164750851556352 BDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚21551132 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚162999059611648 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚48371547 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚61478912 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 125829120 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚4295 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚209 SFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚77261824 NADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚58387428 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚23001716 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚186990866333696 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚322 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 122905600 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚164750774294528 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚187537719363205 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚188647823187968 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚48375842 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚25374126 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚186990866333696 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚322 EDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚48375842 EDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521206 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 $ _timestamp‚1.786660499395006e+09 NGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚82 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚187537777750633 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 1653507486313 trainer/global_step‚6849 K>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59053056 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 1653466138245 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚22997423 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚25374124 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 5494538240 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 D=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚60 H@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚665 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚20643840 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0³ >I2»- _wandbruntime‚1847  _runtime‚1847HHt¼î % _timestamp‚1.7866604993962188e+09 _step‚564 /loss/FRt1/detection_loss‚0.5368506520222116 0loss/FRt2/detection_loss‚0.47821700043640614 epoch‚17 _runtime‚1847.926094735 0loss/FRt1/assignment_loss‚0.7118757042663376 0loss/FRt2/assignment_loss‚0.6315146388006356 &loss/total_loss‚2.3584579955255913 trainer/global_step‚6849ôJh`y½ó _step‚564 /loss/FRt1/detection_loss‚0.5368506520222116 0loss/FRt2/detection_loss‚0.47821700043640614 epoch‚17 _runtime‚1847.926094735 0loss/FRt1/assignment_loss‚0.7118757042663376 0loss/FRt2/assignment_loss‚0.6315146388006356 &loss/total_loss‚2.3584579955255913 trainer/global_step‚6849 % _timestamp‚1.7866604993962188e+09´§Ä–¾‚(:‹ ŸùÓÙˆˆá gpu.0.gpu‚64.0 gpu.0.memory‚12, gpu.0.memoryAllocated‚7.0313262939453125) gpu.0.memoryAllocatedBytes‚ 6039863296 gpu.0.temp‚34.0 gpu.0.powerWatts‚193.146' gpu.0.enforcedPowerLimitWatts‚500.0 gpu.0.powerPercent‚38.6292 gpu.0.smClock‚1410 gpu.0.memoryClock‚1593$ gpu.0.correctedMemoryErrors‚1955# gpu.0.uncorrectedMemoryErrors‚06¿1KŸ¿‚(:” ŸùÓ¶›Íã disk.dm-1.in‚17790.28515625 disk.dm-0.in‚ 43267.5546875 disk.sda1.in‚0 disk.sda1.out‚0 disk.drbd2165.out‚0 disk.sda3.in‚17790.28515625# disk.nvme0c0n1.in‚ 43267.5546875* proc.memory.percent‚14.168477058410645) disk./.usagePercent‚32.73492075686149 disk.sda2.in‚0 disk.dm-1.out‚ 36.36328125 disk.sda.in‚17790.28515625) proc.memory.availableMB‚ 5436.30078125# proc.memory.rssMB‚ 2321.36328125 cpu‚27.074939094571842 proc.cpu.threads‚17 network.sent‚ 9.14962e+06 network.recv‚ 3.657585e+06% disk./.usageGB‚487.77397537231445 disk.drbd2165.in‚0 disk.sda3.out‚ 36.36328125( disk.nvme0c0n1.out‚53737.01318359375 disk.sda.out‚ 36.36328125$ memory_percent‚66.81945323944092# disk.dm-0.out‚53737.01318359375 disk.sda2.out‚0! disk.nvme0n1.in‚ 43267.5546875& disk.nvme0n1.out‚53737.01318359375tm8)2À- _wandbruntime‚1860  _runtime‚1860=?ÄÁ¾ _step‚565 % lr-AdamW/pg1‚0.001128568691418263 % lr-AdamW/pg2‚0.001128568691418263 trainer/global_step‚6899 $ _timestamp‚1.786660511860818e+09 _runtime‚1860.390856262«,ÔÉÂà % lr-AdamW/pg2‚0.001128568691418263 trainer/global_step‚6899 $ _timestamp‚1.786660511860818e+09 _runtime‚1860.390856262 _step‚565 % lr-AdamW/pg1‚0.001128568691418263µ¡c2Ã- _wandbruntime‚1860  _runtime‚1860Wu¼'ĶO IDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 1668312548352 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚262 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚25548223 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚4287 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17104896 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 1763794072064 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚262 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 125829120 @7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚5264 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚165892130912768 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚164128380878848 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 1664838724261 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚322 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚25548221 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚20643840 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 125829120 IBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚60 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 122905600 PIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚65 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚165892195594752 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 831258624 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 1664797376241 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚61478912 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚188832212871845 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 4552982528 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 122905600 D;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚5264 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 4628521206 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚262 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚188832154484465 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 845178880 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 4654602240 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚15905777 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚4289 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚189948973433856 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 1668270603264 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚25548223 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚164128401522688 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚41348020 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 5494538240 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚322 E8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚48707459 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚23163525 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚44038144 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚322 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 5494538240 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 5368709120 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚322 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚188280660885504 PCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚59049984 BDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚189948914383872 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4289 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚23163525 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 M@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚59049984 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 5368709120 G>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4876 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚41945088 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚5793493 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚429 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 122306526 KDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚60 F?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚60 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚23159238 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚41945088 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚15905775 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 5368709120 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 1668312548352 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 1668270603264 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17104896 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚21699270 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚187167357108224 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚5793428 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 trainer/global_step‚6899 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚23159238 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚209 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 1763750033920 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 _step‚566 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚189948914383872 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚187167374147584 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚25548221 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 4654602240 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚188280660885504 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚48711748 IDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚189948914383872 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4289 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚23163525 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 M@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚59049984 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 5368709120 G>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4876 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚41945088 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚5793493 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚429 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 122306526 KDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚60 F?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚60 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚23159238 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚41945088 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚15905775 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 5368709120 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 1668312548352 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 1668270603264 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17104896 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚21699270 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚187167357108224 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚5793428 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 trainer/global_step‚6899 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚23159238 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚209 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 1763750033920 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 _step‚566 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚189948914383872 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚187167374147584 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚25548221 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 4654602240 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚188280660885504 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚48711748 IDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 1668312548352 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚262 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚25548223 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚4287 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17104896 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 1763794072064 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚262 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 125829120 @7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚5264 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚165892130912768 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚164128380878848 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 1664838724261 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚322 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚25548221 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚20643840 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 125829120 IBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚60 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 122905600 PIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚65 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚165892195594752 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 831258624 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 1664797376241 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚61478912 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚188832212871845 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 4552982528 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 122905600 D;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚5264 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 4628521206 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚262 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚188832154484465 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 845178880 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 4654602240 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚15905777 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0¶œƒ ¼2Æ- _wandbruntime‚1860  _runtime‚1860ZXl‘ÇÁM T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚189975526843392 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 831258624 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚165915420498432 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚322 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521206 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 5494538240 G@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚83 K>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59053056 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚25551776 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚189975467790336 H@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚665 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚25551774 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚164151448895488 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 1665027948853 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚209 PDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚21702225 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 845178880 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 1763992246784 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚262 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚188306966118400 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚48714299 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 NADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚59053056 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚188858629238041 NADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚58387428 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚188858570850613 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 5368709120 % _timestamp‚1.7866605120253153e+09 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 122905600 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚21702308 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚5794245 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 1668501671936 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚48718594 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 1668501671936 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚25551776 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 I@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚4876 SFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚70970368 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚322 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚262 D=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚60 _runtime‚1860.555646166 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚23166818 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚188306966118400 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚41948160 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚20643840 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚164151428251648 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 EDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 125829120 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17104896 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚189975526843392 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 1764042573312 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 5368709120 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654602240 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚165915491468800 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 122306526 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚4293 _step‚567 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚48714299 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚23162525 EDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 4552982528 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 122905600 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚50326528 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚61478912 D5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5264 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚4293 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚5794326 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 trainer/global_step‚6899 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚15907982 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚188306983223296 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 1668543620096 NGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚81 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 4517085184 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 5494538240 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚322 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚23162525 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚15907980 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 1665069296921 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 5494538240 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 125829120 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 4654602240 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚41948160 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚262 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚25551774¼†#Ì&ÈÆM VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚15907982 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚188306983223296 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 1668543620096 NGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚81 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 4517085184 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 5494538240 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚322 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚23162525 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚15907980 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 1665069296921 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 5494538240 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 125829120 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 4654602240 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚41948160 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚262 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚25551774 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚189975526843392 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 831258624 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚165915420498432 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚322 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521206 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 5494538240 G@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚83 K>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59053056 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚25551776 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚189975467790336 H@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚665 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚25551774 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚164151448895488 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 1665027948853 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚209 PDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚21702225 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 845178880 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 1763992246784 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚262 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚188306966118400 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚48714299 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 NADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚59053056 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚188858629238041 NADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚58387428 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚188858570850613 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 5368709120 % _timestamp‚1.7866605120253153e+09 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 122905600 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚21702308 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚5794245 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 1668501671936 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚48718594 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 1668501671936 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚25551776 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 I@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚4876 SFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚70970368 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚322 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚262 D=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚60 _runtime‚1860.555646166 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚23166818 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚188306966118400 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚41948160 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚20643840 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚164151428251648 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 EDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 125829120 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17104896 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚189975526843392 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 1764042573312 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 5368709120 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654602240 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚165915491468800 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 122306526 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚4293 _step‚567 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚48714299 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚23162525 EDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 4552982528 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 122905600 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚50326528 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚61478912 D5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5264 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚4293 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚5794326 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 trainer/global_step‚6899·y/‡2É- _wandbruntime‚1860  _runtime‚1860ÍŸwšsÊí 0loss/FRt1/assignment_loss‚0.6832769888670138 0loss/FRt2/assignment_loss‚0.6325009123241182 /loss/FRt1/detection_loss‚0.4978454230944626 0loss/FRt2/detection_loss‚0.48599055181294404 &loss/total_loss‚2.2996138760985385 epoch‚17 trainer/global_step‚6899 $ _timestamp‚1.786660512026353e+09 _runtime‚1860.556144097 _step‚568ÃÇxËò $ _timestamp‚1.786660512026353e+09 _runtime‚1860.556144097 _step‚568 0loss/FRt1/assignment_loss‚0.6832769888670138 0loss/FRt2/assignment_loss‚0.6325009123241182 /loss/FRt1/detection_loss‚0.4978454230944626 0loss/FRt2/detection_loss‚0.48599055181294404 &loss/total_loss‚2.2996138760985385 epoch‚17 trainer/global_step‚6899¸š-z2Ì- _wandbruntime‚1872  _runtime‚1872][ºÆÍÀ trainer/global_step‚6949 % _timestamp‚1.7866605243115003e+09 _runtime‚ 1872.84152656 _step‚569 & lr-AdamW/pg1‚0.0011233677880595509 & lr-AdamW/pg2‚0.0011233677880595509#Ë=üËÎÅ & lr-AdamW/pg2‚0.0011233677880595509 trainer/global_step‚6949 % _timestamp‚1.7866605243115003e+09 _runtime‚ 1872.84152656 _step‚569 & lr-AdamW/pg1‚0.0011233677880595509¹îd´02Ï- _wandbruntime‚1872  _runtime‚1872& …å½'зO \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚20643840 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 1776056291840 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 F?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚60 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚23324340 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 122905600 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚167056761772544 _runtime‚1872.842913712 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚16016025 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚4287 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 1776012253696 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚190153005971873 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 BDeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4289 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 4517085184 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚191276618039296 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 122905600 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚5834321 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 5368709120 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 4654602240 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 831258624 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 5368709120 PCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚59049984 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 1679899314176 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚21850346 PIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚65 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚61478912 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 125829120 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚189596760670208 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚191276618039296 IBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚67 IBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚60 E8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚49050211 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚49054500 KBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚4876 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚16016027 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚25725871 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚322 IDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 G>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4876 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚189596760670208 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 1679857369088 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 125829120 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 4552982528 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚23328627 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚165280749518848 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚262 _step‚570 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚188476646785024 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 1676359186849 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 125829120 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 1679857369088 UHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚64681984 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 1676400534869 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚322 @7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚5264 PCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚58387380 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 5494538240 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 4654602240 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 4552982528 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 845178880 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚41348020 IDeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4289 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 4517085184 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚191276618039296 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 122905600 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚5834321 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 5368709120 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 4654602240 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 831258624 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 5368709120 PCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚59049984 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 1679899314176 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚21850346 PIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚65 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚61478912 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 125829120 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚189596760670208 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚191276618039296 IBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚67 IBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚60 E8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚49050211 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚49054500 KBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚4876 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚16016027 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚25725871 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚322 IDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 G>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4876 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚189596760670208 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 1679857369088 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 125829120 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 4552982528 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚23328627 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚165280749518848 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚262 _step‚570 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚188476646785024 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 1676359186849 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 125829120 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 1679857369088 UHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚64681984 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 1676400534869 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚322 @7DeviceStatsMonitor.on_train_batch_start/active.allvŒ15à.peak‚5264 PCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚58387380 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 5494538240 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 4654602240 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 4552982528 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 845178880 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚41348020 IDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 % _timestamp‚1.7866605244833474e+09 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 1680130385920 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚21853387 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚167080119182848 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚20643840 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 125829120 D=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚60 trainer/global_step‚6949 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚429 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 1676589759461 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚25729424 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚25729426 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚23327627 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚165303817535488 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚188502832578560 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚49061346 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚16018230 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 122306526 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 1680130385920 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 P6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚322 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚191303230498816 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 5494538240 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚189623083008000 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚5835157 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚41348068 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 125829120 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚23327627 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17104896 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 122905600 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚49057051 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 5368709120 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚322 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚25729424 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚4293 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 5494538240 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚191303230498816 K>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59053056 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚25729426 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚4293 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚5835222 NGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚65 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 122905600 EDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521206 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚188502849617920 NADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚59053056 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚41948160 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚190179480725449 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 4552982528 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 4552982528 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚46132224 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚49057051 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚189623100112896 H@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚665 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 5368709120 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17104896 B9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚5264 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 125829120 >5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5264 I@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚4876 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚322 EDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654602240 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚49061346 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚262 IBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚60 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚4295 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚23331920 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 5368709120 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚262 _runtime‚1873.013995124 D8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 P6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚322 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚191303230498816 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 5494538240 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚189623083008000 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚5835157 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚41348068 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 125829120 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚23327627 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17104896 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 122905600 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚49057051 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 5368709120 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚322 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚25729424 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚4293 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 5494538240 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚191303230498816 K>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59053056 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚25729426 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚4293 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚5835222 NGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚65 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 122905600 EDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521206 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚188502849617920 NADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚59053056 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚41948160 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚190179480725449 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 4552982528 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 4552982528 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚46132224 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚49057051 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚189623100112896 H@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚665 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 5368709120 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17104896 B9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚5264 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 125829120 >5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5264 I@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚4876 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚322 EDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654602240 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚49061346 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚262 IBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚60 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚4295 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚23331920 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 5368709120 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚262 _runtime‚1873.013995124 DDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 % _timestamp‚1.7866605244833474e+09 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 1680130385920 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚21853387 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚167080119182848 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚20643840 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 125829120 D=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚60 trainer/global_step‚6949 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚429 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 1676589759461»:†MÞ2Õ- _wandbruntime‚1873  _runtime‚1873 —WîrÖì % _timestamp‚1.7866605244844956e+09 _step‚572 0loss/FRt1/assignment_loss‚0.7831021461653741 0loss/FRt2/assignment_loss‚0.6654934499639051 %loss/total_loss‚2.452806044198596 trainer/global_step‚6949 _runtime‚1873.014752055 /loss/FRt1/detection_loss‚0.5608752983709311 /loss/FRt2/detection_loss‚0.4433351496983857 epoch‚17žã2ow×ñ _runtime‚1873.014752055 /loss/FRt1/detection_loss‚0.5608752983709311 /loss/FRt2/detection_loss‚0.4433351496983857 epoch‚17 % _timestamp‚1.7866605244844956e+09 _step‚572 0loss/FRt1/assignment_loss‚0.7831021461653741 0loss/FRt2/assignment_loss‚0.6654934499639051 %loss/total_loss‚2.452806044198596 trainer/global_step‚6949¼j‡— ŸØ‚(:” ®ùÓ›‚–á gpu.0.gpu‚20.0 gpu.0.memory‚2, gpu.0.memoryAllocated‚7.0313262939453125) gpu.0.memoryAllocatedBytes‚ 6039863296 gpu.0.temp‚34.0 gpu.0.powerWatts‚220.656' gpu.0.enforcedPowerLimitWatts‚500.0( gpu.0.powerPercent‚44.13120000000001 gpu.0.smClock‚1410 gpu.0.memoryClock‚1593$ gpu.0.correctedMemoryErrors‚1955# gpu.0.uncorrectedMemoryErrors‚0¤[0Ò–Ù‚(:‹ ®ùÓ¶éã! disk.dm-0.out‚53741.798828125& disk.nvme0c0n1.out‚53741.798828125 disk.sda1.in‚0 disk.sda1.out‚0 network.sent‚ 9.224105e+06 disk.dm-1.out‚ 36.5859375 disk.drbd2165.in‚0 disk.drbd2165.out‚0 disk.sda3.out‚ 36.5859375 disk.sda.out‚ 36.5859375) proc.memory.availableMB‚ 5435.87109375* proc.memory.percent‚14.168477058410645$ disk.nvme0n1.out‚53741.798828125 disk.dm-1.in‚17790.28515625 disk.dm-0.in‚ 43267.5546875# disk.nvme0c0n1.in‚ 43267.5546875 disk.sda2.in‚0 disk.sda2.out‚0 proc.cpu.threads‚17 network.recv‚ 3.689469e+06$ disk./.usageGB‚487.7685775756836 disk.sda3.in‚17790.28515625 disk.sda.in‚17790.28515625$ memory_percent‚66.82207584381104# proc.memory.rssMB‚ 2321.36328125 cpu‚27.074998389950704) disk./.usagePercent‚32.73455850620054! disk.nvme0n1.in‚ 43267.5546875Þ Ã2Ú- _wandbruntime‚1885  _runtime‚1885Çî<ÅÛ¿ trainer/global_step‚6999 % _timestamp‚1.7866605372343705e+09 _runtime‚1885.764378887 _step‚573 % lr-AdamW/pg1‚0.001118142903985506 % lr-AdamW/pg2‚0.001118142903985506ÓºœÊÜÄ % lr-AdamW/pg1‚0.001118142903985506 % lr-AdamW/pg2‚0.001118142903985506 trainer/global_step‚6999 % _timestamp‚1.7866605372343705e+09 _runtime‚1885.764378887 _step‚573½%@gÙ2Ý- _wandbruntime‚1885  _runtime‚1885sñGˆ¶Þ¶O GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 5368709120 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 125829120 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 4654602240 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚25903521 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 5368709120 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚191473915846661 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 4517085184 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚322 PCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚58387380 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚189785953501184 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚189785936461824 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 125829120 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚23493729 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚61478912 IBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚60 PCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚59049984 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚429 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚23489442 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 1691444134912 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚166433138802688 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚262 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚23493729 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17104896 D;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚5264 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚22001721 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚223 PIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚65 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 125829120 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 trainer/global_step‚6999 JBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚665 KBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚4876 BDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 122905600 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 845178880 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚41945088 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚190912894664704 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 5494538240 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚322 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚190912894664704 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 1691486080000 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 1691486080000 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚4287 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 1687920997457 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 4552982528 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚22001654 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚168221428947456 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 1687962345477 @7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚5264 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚49397252 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4289 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚25903521 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚192604380744704 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚25903523 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚44038144 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 _runtime‚1#Ë 1885.768926618 G>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4876 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚192604321694720 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚16126277 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 122306526 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚190912877559808 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚5875444 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 4552982528 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚262 KDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚60 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 1691444134912 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚192604321694720 IDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚192604321694720 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚16126277 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 122306526 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚190912877559808 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚5875444 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 4552982528 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚262 KDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚60 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 1691444134912 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚192604321694720 IDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 122905600 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 845178880 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚41945088 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚190912894664704 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 5494538240 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚322 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚190912894664704 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 1691486080000 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 1691486080000 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚4287 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 1687920997457 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 4552982528 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚22001654 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚168221428947456 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 1687962345477 @7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚5264 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚49397252 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4289 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚25903521 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚192604380744704 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚25903523 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚44038144 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 _runtime‚1885.768926618 G>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4876¾aÖšÕ2à- _wandbruntime‚1885  _runtime‚1885ºú‚LÇ&áÁM QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 4552982528 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚49404098 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 1788553001472 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 IBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚60 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚5876275 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 122905600 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚189812139294720 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 125829120 % _timestamp‚1.7866605374160588e+09 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚23497022 EDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚49404098 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚192630934154240 K>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59053056 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 4552982528 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 5368709120 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 125829120 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚262 B9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚5264 NGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚65 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚166456165531648 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521206 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 122905600 _runtime‚1885.946618796 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚41948160 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚41348068 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 1688151570069 G@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚60 D=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚60 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚25907076 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 1691675203584 NADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚58387428 DDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654602240 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 1691675203584 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚23497022 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚16128480 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 5368709120 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚322 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚192630934154240 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 _step‚575 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 4517085184 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 5494538240 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚22004757 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 845178880 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚191500332212857 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 5494538240 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 1691717151744 SFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚66776064 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚20643840 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚192630875101184 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 trainer/global_step‚6999 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚190939217002496 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚190939199897600 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚322 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚4293 E5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5264 BDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚22004690 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 1688192918137 NADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚59053056 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚168244785309184 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 5494538240 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚23492729 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚166456186175488 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚25907074 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚16128482 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚322 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0333IÌ&âÆM ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 5494538240 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 1691717151744 SFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚66776064 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚20643840 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚192630875101184 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 trainer/global_step‚6999 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚190939217002496 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚190939199897600 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚322 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚4293 E5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5264 BDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚22004690 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 1688192918137 NADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚59053056 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚168244785309184 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 5494538240 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚23492729 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚166456186175488 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚25907074 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚16128482 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚322 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 4552982528 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚49404098 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 1788553001472 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 IBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚60 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚5876275 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 122905600 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚189812139294720 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 125829120 % _timestamp‚1.7866605374160588e+09 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚23497022 EDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚49404098 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚192630934154240 K>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59053056 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 4552982528 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 5368709120 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 125829120 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚262 B9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚5264 NGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚65 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚166456165531648 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521206 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 122905600 _runtime‚1885.946618796 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚41948160 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚41348068 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 1688151570069 G@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚60 D=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚60 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚25907076 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 1691675203584 NADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚58387428 DDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654602240 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 1691675203584 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚23497022 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚16128480 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 5368709120 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚322 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚192630934154240 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 _step‚575 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 4517085184 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 5494538240 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚22004757 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 845178880 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚191500332212857¿¡Ÿ2ã- _wandbruntime‚1885  _runtime‚1885`[äë _runtime‚1885.947182191 _step‚576 /loss/FRt1/assignment_loss‚0.740278043408056 0loss/FRt2/assignment_loss‚0.6535583281967693 /loss/FRt1/detection_loss‚0.5520022995370509 &loss/total_loss‚2.4756683743095436 trainer/global_step‚6999 /l£w+C`oss/FRt2/detection_loss‚0.5298297031676673 epoch‚17 $ _timestamp‚1.786660537417298e+09^z¶våð /loss/FRt1/assignment_loss‚0.740278043408056 0loss/FRt2/assignment_loss‚0.6535583281967693 /loss/FRt1/detection_loss‚0.5520022995370509 &loss/total_loss‚2.4756683743095436 trainer/global_step‚6999 /loss/FRt2/detection_loss‚0.5298297031676673 epoch‚17 $ _timestamp‚1.786660537417298e+09 _runtime‚1885.947182191 _step‚576À‚ÐΡæ‚(:– ½ùÓéüœá gpu.0.gpu‚62.0 gpu.0.memory‚12, gpu.0.memoryAllocated‚7.0313262939453125) gpu.0.memoryAllocatedBytes‚ 6039863296 gpu.0.temp‚35.0 gpu.0.powerWatts‚122.542' gpu.0.enforcedPowerLimitWatts‚500.0) gpu.0.powerPercent‚24.508399999999998 gpu.0.smClock‚1410 gpu.0.memoryClock‚1593$ gpu.0.correctedMemoryErrors‚1955# gpu.0.uncorrectedMemoryErrors‚0ÖšÔÅç‚(:’ ½ùÓŒ€åã disk.sda3.out‚ 36.71484375# disk.nvme0c0n1.in‚ 43267.5546875 disk.sda.out‚ 36.71484375 cpu‚27.075453431142062 disk.drbd2165.out‚0 network.recv‚ 3.718506e+06% disk./.usageGB‚487.76952362060547! disk.nvme0n1.in‚ 43267.5546875 disk.dm-0.in‚ 43267.5546875" disk.dm-0.out‚53788.1474609375 disk.sda3.in‚17790.28515625' disk.nvme0c0n1.out‚53788.1474609375% disk.nvme0n1.out‚53788.1474609375 disk.sda.in‚17790.28515625$ memory_percent‚66.82164669036865) proc.memory.availableMB‚ 5435.94140625# proc.memory.rssMB‚ 2321.36328125* proc.memory.percent‚14.168477058410645 proc.cpu.threads‚17) disk./.usagePercent‚32.73462199606904 disk.sda1.out‚0 disk.sda2.in‚0 disk.sda2.out‚0 disk.drbd2165.in‚0 network.sent‚ 9.315864e+06 disk.sda1.in‚0 disk.dm-1.in‚17790.28515625 disk.dm-1.out‚ 36.71484375°s\:2è- _wandbruntime‚1898  _runtime‚1898ܶwÅé¿ % lr-AdamW/pg1‚0.001112894374780669 % lr-AdamW/pg2‚0.001112894374780669 trainer/global_step‚7049 % _timestamp‚1.7866605497572293e+09 _runtime‚1898.287263673 _step‚577¤×3ÊêÄ _step‚577 % lr-AdamW/pg1‚0.001112894374780669 % lr-AdamW/pg2‚0.001112894374780669 trainer/global_step‚7049 % _timestamp‚1.7866605497572293e+09 _runtime‚1898.287263673ÁÄÅp2ë- _wandbruntime‚1898  _runtime‚1898èœâ¼'ì¶O M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚49740004 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚23658831 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚41945088 @7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚5264 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚61478912 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 IBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚63 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚26081171 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17104896 D;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚5264 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚322 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚192228994449408 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 4552982528 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 1703072845824 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 1800603187712 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 5368709120 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 1800649323008 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 M@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚59049984 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 4517085184 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 125829120 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚322 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚192229011554304 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 1703072845824 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚16236527 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚5916406 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 125829120 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 KBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚4876 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚262 BDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 1699482808065 F?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚60 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚16236525 E8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚49735715 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 122905600 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚26081173 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚192794767334069 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 4628521206 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚191095243177984 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 125829120 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 5494538240 IDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4876 C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚4289 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 KDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚60 _runtime‚ 1898.28846459 PIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚61 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚193932084400128 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 831258624 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 5494538240 trainer/global_step‚7049 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚223 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 1703030900736 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚20643840 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚262 PCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚59049984 IDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 5368709120 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚4287 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚193932084400128 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚41945088 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚322 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 4654602240 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚167585486798848 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚192228994449408 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 4552982528 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 1703030900736 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4289 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 PCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚58387380 JBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚665 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚193932025350144 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚192229011554304 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17104896 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚5916345 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚41348020 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 5368709120 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚209 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 1699524156085 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 5494538240Gd¹Á'í»O [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 1703072845824 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚16236527 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚5916406 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 125829120 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 KBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚4876 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚262 BDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 1699482808065 F?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚60 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚16236525 E8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚49735715 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 122905600 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚26081173 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚192794767334069 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 4628521206 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚191095243177984 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 125829120 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 5494538240 IDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4876 C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚4289 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 KDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚60 _runtime‚ 1898.28846459 PIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚61 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚193932084400128 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 831258624 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 5494538240 trainer/global_step‚7049 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚223 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 1703030900736 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚20643840 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚262 PCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚59049984 IDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 5368709120 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚4287 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚193932084400128 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚41945088 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚322 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 4654602240 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚167585486798848 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚192228994449408 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 4552982528 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 1703030900736 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4289 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 PCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚58387380 JBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚665 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚193932025350144 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚192229011554304 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17104896 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚5916345 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚41348020 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 5368709120 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚209 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 1699524156085 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 5494538240 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚49740004 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚23658831 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚41945088 @7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚5264 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚61478912 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 IBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚63 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚26081171 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17104896 D;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚5264 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚322 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚192228994449408 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 4552982528 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 1703072845824 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 1800603187712 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 5368709120 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 1800649323008 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 M@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚59049984 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 4517085184 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 125829120 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚322 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚192229011554304ÂXn¯2î- _wandbruntime‚1898  _runtime‚1898Œ7DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 1703261969408 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 122905600 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 1800894514688 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚26084724 EDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521206 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚322 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚41948160 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚192255316787200 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚193958637809664 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚322 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚192821183700265 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 1699754728745 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚262 DDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚26084724 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚23657831 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654602240 B9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚5264 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 1800850479616 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 122306526 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17104896 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 831258624 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚61478912 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚262 D=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚60 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 1703303917568 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚49746850 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚26084726 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚16238730 NGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚70 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 5494538240 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 5494538240 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 5368709120 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚49746850 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚167608554815488 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 trainer/global_step‚7049 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚4295 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 122905600 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚4293 IBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚60 >5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5264 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 4654602240 H@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚665 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚167608534171648 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚23657831 G@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚72 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚209 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚322 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚22155846 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 4517085184 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚429 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚192821125312837 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 4552982528 SFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚64678912 _runtime‚1898.458615908 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚192255333892096 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 4552982528 EDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59053056 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0_ÀZæ(ðÆM [GDeviceStatsMonitor.on_train_ba/Òv¤&tch_end/active_bytes.large_pool.allocated‚192255333892096 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17104896 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 1703303917568 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚5917116 BDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59053056 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚26084726 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚169409384651264 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 5368709120 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚23662124 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 125829120 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 125829120 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 1703261969408 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 122905600 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 1800894514688 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚26084724 EDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521206 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚322 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚41948160 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚192255316787200 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚193958637809664 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚322 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚192821183700265 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 1699754728745 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚262 DDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚26084724 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚23657831 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654602240 B9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚5264 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 1800850479616 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 122306526 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17104896 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 831258624 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚61478912 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚262 D=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚60 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 1703303917568 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚49746850 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚26084726 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚16238730 NGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚70 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 5494538240 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 5494538240 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 5368709120 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚49746850 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚167608554815488 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 trainer/global_step‚7049 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚4295 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 122905600 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚4293 IBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚60 >5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5264 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 4654602240 H@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚665 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚167608534171648 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚23657831 G@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚72 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚209 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚322 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚22155846 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 4517085184 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚429 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚192821125312837 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 4552982528 SFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚64678912 _runtime‚1898.458615908 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚192255333892096 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 4552982528 EDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚5957143 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 5494538240 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 5494538240 G>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4876 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚193545111339008 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 122905600 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 845178880 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚44038144 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 122306526 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 5494538240 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚23819646 KBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚4876 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 1711044618673 trainer/global_step‚7099 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4289 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚429 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚194115618821477 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚16346775 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚195259729005568 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17104896 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 1714617666560 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 4654602240 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚26258821 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚16346777 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚194115560434097 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17104896 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚41945088 UHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚64681984 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 5368709120 PCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚58387380 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚4287 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 IBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚60 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 $ _timestamp‚1.786660562427334e+09 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚26258823 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚193545111339008 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚26258821 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚195259729005568 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 4628521206 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚322 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 1812951461376 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚192404515815424 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 5368709120 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚262 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚22303920 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 1714617666560 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚50082756 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚170550762862080 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 4654602240 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 125829120 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 125829120 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚322 C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚4289 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚22303858 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚5957083 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚41348020 _step‚582 M@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚59049984 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 5368709120 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚193545128443904 D;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚5264 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚322 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚23819646ìÿþ³'û»O N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 5494538240 G>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4876 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚193545111339008 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 122905600 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 845178880 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚44038144 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 122306526 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 5494538240 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚23819646 KBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚4876 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 1711044618673 trainer/global_step‚7099 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4289 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚429 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚194115618821477 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚16346775 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚195259729005568 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17104896 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 1714617666560 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 4654602240 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚26258821 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚16346777 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚194115560434097 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17104896 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚41945088 UHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚64681984 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 5368709120 PCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚58387380 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚4287 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 IBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚60 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 $ _timestamp‚1.786660562427334e+09 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚26258823 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚193545111339008 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚26258821 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚195259729005568 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 4628521206 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚322 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 1812951461376 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚192404515815424 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 5368709120 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚262 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚22303920 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 1714617666560 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚50082756 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚170550762862080 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 4654602240 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 125829120 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 125829120 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚322 C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚4289 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚22303858 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚5957083 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚41348020 _step‚582 M@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚59049984 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 5368709120 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚193545128443904 D;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚5264 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚322 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚23819646 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 JBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚665 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚168737876082688 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚20643840 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 1714659611648 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 4552982528 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚61478912 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 4517085184 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚26258823 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚23823933 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 1812907423232 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 122905600 PCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚59049984 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚209 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 _runtime‚1910.960919739 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚41945088 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚4287 PIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚60 E8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚50078467 IBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚62 @7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚5264 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚168737855438848 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 1711085966693 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚262 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚262 KDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚60 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚195259788055552 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚195259788055552 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚223 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 4552982528 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚23823933 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 125829120 F?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚60 IDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.all‚}¢´±ocated‚5957143 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 5494538240Æ›Á±ˆ2ü- _wandbruntime‚1911  _runtime‚1911ý«ïÇ&ýÁM UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 1714848735232 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚322 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚23822933 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚194142035187673 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚194141976800245 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521206 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚192430718648320 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17104896 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚23822933 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚262 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 125829120 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚41348068 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 5494538240 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 BDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚50089602 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚22306844 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚209 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17104896 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 122306526 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 5494538240 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 5368709120 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚170574049826304 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 NADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚58387428 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 4517085184 G@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚60 _runtime‚1911.125089815 NADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚59053056 PDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚4293 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚5957864 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 1813197341184 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚223 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 4654602240 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚193571450781696 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚26262374 NGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚71 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚26262376 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 122905600 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚168760902811648 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚50326528 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 1711316539353 % _timestamp‚1.7866605625947301e+09 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 125829120 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚195286341465088 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚50085307 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚16348980 K>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59053056 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚262 B9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚5264 I@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚4876 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 831258624 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚192430701608960 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 5368709120 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚262 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654602240 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚41948160 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚4293 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚41948160 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 1714848735232 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚23827226 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 1714890683392 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚20643840 >5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5264 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 1714890683392 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚322 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 125829120 IBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚60 E6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚322 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚195286282412032 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚168760923455488 D=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚60 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚50085307 EDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚4293 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚5957864 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 1813197341184 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚223 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 4654602240 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚193571450781696 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚26262374 NGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚71 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚26262376 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 122905600 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚168760902811648 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚50326528 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 1711316539353 % _timestamp‚1.7866605625947301e+09 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 125829120 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚195286341465088 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚50085307 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚16348980 K>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59053056 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚262 B9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚5264 I@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚4876 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 831258624 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚192430701608960 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 5368709120 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚262 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654602240 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚41948160 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚4293 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚41948160 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 1714848735232 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚23827226 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 1714890683392 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚20643840 >5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5264 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 1714890683392 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚322 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 125829120 IBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚60 E6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚322 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚195286282412032 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚168760923455488 D=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚60 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚50085307 EDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521206 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚192430718648320 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17104896 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚23822933 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚262 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 125829120 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚41348068 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 5494538240 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 BDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚50089602 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚22306844 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚209 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0Çï¶(Â2ÿ- _wandbruntime‚1911  _runtime‚1911´äŽr€ì /loss/FRt2/detection_loss‚0.5008774317824286 epoch‚17 trainer/global_step‚7099 _step‚584 0loss/FRt2/assignment_loss‚0.5332576369159179 %loss/total_loss‚2.222672114704038 % _timestamp‚1.7866605625958834e+09 _runtime‚1911.125784637 0loss/FRt1/assignment_loss‚0.6708094597749209 /loss/FRt1/detection_loss‚0.5177275862307704êÂwñ 0loss/FRt1/assignment_loss‚0.6708094597749209 /loss/FRt1/detection_loss‚0.5177275862307704 /loss/FRt2/detection_loss‚0.5008774317824286 epoch‚17 trainer/global_step‚7099 _step‚584 0loss/FRt2/assignment_loss‚0.5332576369159179 %loss/total_loss‚2.222672114704038 % _timestamp‚1.7866605625958834e+09 _runtime‚1911.125784637ÈÁ (û–‚‚(:‹ ÛùÓÆù¦á gpu.0.gpu‚64.0 gpu.0.memory‚12, gpu.0.memoryAllocated‚7.0313262939453125) gpu.0.memoryAllocatedBytes‚ 6039863296 gpu.0.temp‚35.0 gpu.0.powerWatts‚221.957' gpu.0.enforcedPowerLimitWatts‚500.0 gpu.0.powerPercent‚44.3914 gpu.0.smClock‚1410 gpu.0.memoryClock‚1593$ gpu.0.correctedMemoryErrors‚1955# gpu.0.uncorrectedMemoryErrors‚05Øk#Œƒ‚(: ÛùÓý­–ã disk.dm-1.in‚ 18835.0546875# proc.memory.rssMB‚ 2321.36328125 cpu‚27.075719414835504 disk.sda3.in‚18835.80078125 disk.sda3.out‚37.125 disk.sda1.in‚0 disk.sda1.out‚0" disk.nvme0n1.in‚43278.94140625 disk.dm-1.out‚37.125 disk.drbd2165.out‚0! disk.dm-0.out‚53839.982421875 network.recv‚ 3.770657e+06% disk./.usageGB‚487.76940155029297 disk.sda.out‚37.125$ disk.nvme0n1.out‚53839.982421875 disk.drbd2165.in‚0$ memory_percent‚66.82124137878418* proc.memory.percent‚14.168477058410645 proc.cpu.threads‚17 network.sent‚ 9.453878e+06$ disk.nvme0c0n1.in‚43278.94140625& disk.nvme0c0n1.out‚53839.982421875 disk.sda2.in‚0 disk.sda2.out‚0 disk.dm-0.in‚43278.94140625( proc.memory.availableMB‚ 5436.0078125) disk./.usagePercent‚32.73461380382794 disk.sda.in‚18835.80078125Fªf2„- _wandbruntime‚1923  _runtime‚1923%|zÆ…À trainer/global_step‚7149 $ _timestamp‚1.786660574880613e+09 _runtime‚1923.410674817 _step‚585 & lr-AdamW/pg1‚0.0011023277308885439 & lr-AdamW/pg2‚0.0011023277308885439@¦ÿˆŠ$ _timestamp‚1.786660574880613e+09 _runtime‚1923.410674817 _step‚585 & lr-AdamW/pg1‚0.0011023277308885439 & lr-AdamW/pg2‚0.0011023277308885439 trainer/global_step‚7149Éä13,2‡- _wandbruntime‚1923  _runtime‚1923 ýY/ö&ˆ·O E8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚50421219 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 4654602240 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚41945088 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 122905600 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 5368709120 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 125829120 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚322 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚194861228228608 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚196587432660992 IBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚66 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 845178880 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 1825232585728 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 122306526 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 4552982528 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 122905600 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚26436471 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚194861245333504 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚50425508 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 5494538240 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚322 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 4654602240 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 1722606429281 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚26436473 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 125829120 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚223 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 5494538240 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 M@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚59049984 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚169890244722688 BDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚196587432660992 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚22455190 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚209 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 1825190644736 _step‚586 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17104896 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚16457025 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚5998165 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 _runtime‚1923.411987388 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚196587491710976 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚22455256 UHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚62584832 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 831258624 KDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚60 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17104896 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚171715414723584 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 5368709120 @7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚5264 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 1726204432384 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 1726204432384 JBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚665 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚23989035 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚322 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚23984748 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚262 IBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚60 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 1726246377472 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4289 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 5494538240 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚26436471 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚195436470308885 G>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4876 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚4287 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 125829120 IDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 PCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚58387380 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚193713805492224 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚194861245333504 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 4517085184 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 4552982528 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚171715477308416 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚41940992 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚262 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚23984748 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚429 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚262 C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚4289 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚196587491710976 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 1726246377472 F>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚41945088 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚26436473 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚16457027 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚193713822531584 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚23989035 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚194861228228608 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚20643840 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚169890224078848 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚195436411921505 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 4628521206 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 5368709120 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bfùŽÇytes.small_pool.peak‚61478912 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 1722647777301 F?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚60½ÂÂ'‰¼O IDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 PCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚58387380 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚193713805492224 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚194861245333504 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 4517085184 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 4552982528 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚171715477308416 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚41940992 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚262 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚23984748 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚429 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚262 C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚4289 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚196587491710976 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 1726246377472 F>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚41945088 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚26436473 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚16457027 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚193713822531584 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚23989035 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚194861228228608 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚20643840 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚169890224078848 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚195436411921505 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 4628521206 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 5368709120 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚61478912 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 1722647777301 F?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚60 E8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚50421219 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 4654602240 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚41945088 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 122905600 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 5368709120 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 125829120 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚322 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚194861228228608 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚196587432660992 IBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚66 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 845178880 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 1825232585728 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 122306526 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 4552982528 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 122905600 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚26436471 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚194861245333504 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚50425508 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 5494538240 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚322 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 4654602240 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 1722606429281 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚26436473 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 125829120 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚223 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 5494538240 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 M@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚59049984 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚169890244722688 BDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚196587432660992 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚22455190 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚209 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 1825190644736 _step‚586 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17104896 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚16457025 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚5998165 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 _runtime‚1923.411987388 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚196587491710976 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚22455256 UHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚62584832 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 831258624 KDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚60 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17104896 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚171715414723584 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 5368709120 @7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚5264 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 1726204432384 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 1726204432384 JBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚665 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚23989035 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚322 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚23984748 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚262 IBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚60 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 1726246377472 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4289 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 5494538240 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚26436471 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚195436470308885 G>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4876 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚4287 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 125829120 IDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 1726435501056 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 1722837001893 DDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚22458224 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚50432354 SFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚70970368 I@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚4876 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚22458308 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚5998994 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚171738774755328 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 1825482659840 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 125829120 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 125829120 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚322 EDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 4552982528 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521206 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 122905600 D=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚60 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚4293 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 5494538240 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 125829120 >5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5264 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚209 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚322 G@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚60 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 4654602240 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654602240 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚194887567671296 H@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚665 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚195462886675081 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚262 trainer/global_step‚7149 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 1726477449216 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 845178880 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚322 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 IBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚60 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 831258624 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 5368709120 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 _runtime‚1923.576835673 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚50428059 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚16459232 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚193740008325120 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 122306526 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚4295 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚26440026 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚194887550566400 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚193739991285760 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚50432354 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17104896 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚26440024 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚171738703784960 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚169913271451648 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚26440026 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚196613986067456 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 PDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 K>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59053056 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚23988035 [v¶Ì&ŒÆM LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚209 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚322 G@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚60 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 4654602240 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654602240 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚194887567671296 H@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚665 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚195462886675081 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚262 trainer/global_step‚7149 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 1726477449216 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 845178880 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚322 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 IBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚60 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 831258624 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 5368709120 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 _runtime‚1923.576835673 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚50428059 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚16459232 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚193740008325120 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 122306526 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚4295 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚26440026 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚194887550566400 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚193739991285760 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚50432354 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17104896 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚26440024 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚171738703784960 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚169913271451648 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚26440026 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚196613986067456 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 PDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 K>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59053056 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚23988035 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚322 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚5999076 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚4293 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 5368709120 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚262 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 4552982528 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 1726477449216 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚20643840 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚50326528 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 1722878349961 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚262 % _timestamp‚1.7866605750464554e+09 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚26440024 B9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚5264 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚16459230 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 1726435501056 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 1722837001893 DDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚22458224 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚50432354 SFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚70970368 I@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚4876 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚22458308 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚5998994 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚171738774755328 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 1825482659840 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 125829120 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 125829120 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚322 EDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 4552982528 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521206 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 122905600 D=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚60 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚4293 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 5494538240 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 125829120 >5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5264ËH2- _wandbruntime‚1923  _runtime‚1923Üd®AsŽí 0loss/FRt1/assignment_loss‚0.7512040231131181 /loss/FRt2/detection_loss‚0.5155466228866076 &loss/total_loss‚2.4430316870003876 % _timestamp‚1.7866605750479565e+09 _step‚588 0loss/FRt2/assignment_loss‚0.6163393859102109 /loss/FRt1/detection_loss‚0.5599416550904511 epoch‚17 trainer/global_step‚7149 _runtime‚1923.577795232`ƒÌ~xò &loss/total_loss‚2.4430316870003876 % _timestamp‚1.7866605750479565e+09 _step‚588 0loss/FRt2/assignment_loss‚0.6163393859102109 /loss/FRt1/detection_loss‚0.5599416550904511 epoch‚17 trainer/global_step‚7149 _runtime‚1923.577795232 0loss/FRt1/assignment_loss‚0.7512040231131181 /loss/FRt2/detection_loss‚0.5155466228866076̦ÆK"¡‚(:– êùÓÁ’€á gpu.0.gpu‚57.0 gpu.0.memory‚10, gpu.0.memoryAllocated‚7.0313262939453125) gpu.0.memoryAllocatedBytes‚ 6039863296 gpu.0.temp‚35.0 gpu.0.powerWatts‚222.217' gpu.0.enforcedPowerLimitWatts‚500.0) gpu.0.powerPercent‚44.443400000000004 gpu.0.smClock‚1410 gpu.0.memoryClock‚1593$ gpu.0.correctedMemoryErrors‚1955# gpu.0.uncorrectedMemoryErrors‚0Ì#?ˆ‘‚(:ý êùÓ®ØÅä disk.dm-0.in‚ 43285.984375# disk.dm-0.out‚53884.38818359375 disk.sda3.out‚37.625" disk.nvme0c0n1.in‚ 43285.984375 disk.sda2.out‚0$ memory_percent‚66.82178974151611) proc.memory.availableMB‚ 5435.91796875( disk./.usagePercent‚32.7351757403656 disk.drbd2165.in‚0( disk.nvme0c0n1.out‚53884.38818359375 disk.sda.out‚37.625* proc.memory.percent‚14.168477058410645 network.sent‚ 9.523236e+06 disk.dm-1.in‚ 18999.6328125 disk.dm-1.out‚37.625 disk.sda.in‚ 18999.6328125 disk.nvme0n1.in‚ 43285.984375 cpu‚27.075992121549277 proc.cpu.threads‚17# disk./.usageGB‚487.777774810791 disk.drbd2165.out‚0 disk.sda3.in‚ 18999.6328125 disk.sda1.in‚0 disk.sda1.out‚0 disk.sda2.in‚0& disk.nvme0n1.out‚53884.38818359375# proc.memory.rssMB‚ 2321.36328125 network.recv‚ 3.796972e+06Jâ¶2’- _wandbruntime‚1936  _runtime‚1936j‡ô/Ç“Á _step‚589 & lr-AdamW/pg1‚0.0010970102948770469 & lr-AdamW/pg2‚0.0010970102948770469 trainer/global_step‚7199 % _timestamp‚1.7866605876178453e+09 _runtime‚1936.149916016rÛ£”Æ & lr-AdamW/pg1‚œwÿ´0.0010970102948770469 & lr-AdamW/pg2‚0.0010970102948770469 trainer/global_step‚7199 % _timestamp‚1.7866605876178453e+09 _runtime‚1936.149916016 _step‚589ͱ#l¬2•- _wandbruntime‚1936  _runtime‚1936)EC½'–·O JBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚665 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚429 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚172880133180928 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 4628521206 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚195023112208384 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚196177362223104 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 4654602240 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚196177345118208 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 1837519818240 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚322 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚262 % _timestamp‚1.7866605876185532e+09 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 E8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚50763971 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚24149850 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚197915195366400 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 4654602240 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17104896 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚172880072693248 _step‚590 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 PIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚61 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚171042592718848 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 IBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚60 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚196177345118208 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 1737833143296 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚197915136316416 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4289 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 5494538240 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚262 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚41945088 D;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚5264 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚24154137 KBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚4876 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚16567277 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 G>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4876 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚197915195366400 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 5494538240 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚322 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 122905600 IDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚41348020 F?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚60 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚197915136316416 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚196177362223104 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚262 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 125829120 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚22606128 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚20643840 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 125829120 F>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚24149850 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚195023095169024 trainer/global_step‚7199 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚24154137 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚209 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 1734168239889 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 5368709120 KDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚60 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚6038851 UHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚60487680 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚61478912 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 1737833143296¦²‘žÂ'—¼O RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 122905600 IDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚41348020 F?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚60 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚197915136316416 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚196177362223104 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚262 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 125829120 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚22606128 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚20643840 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 125829120 F>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚24149850 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚195023095169024 trainer/global_step‚7199 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚24154137 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚209 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 1734168239889 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 5368709120 KDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚60 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚6038851 UHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚60487680 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚61478912 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 1737833143296 JBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚665 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚429 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚172880133180928 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 4628521206 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚195023112208384 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚196177362223104 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 4654602240 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚196177345118208 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 1837519818240 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚322 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚262 % _timestamp‚1.7866605876185532e+09 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 E8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚50763971 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚24149850 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚197915195366400 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 4654602240 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17104896 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚172880072693248 _step‚590 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 PIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚61 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚171042592718848 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 IBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚60 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚196177345118208 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 1737833143296 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚197915136316416 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4289 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 5494538240 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚262 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚41945088 D;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚5264 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚24154137 KBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚4876 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚16567277 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 G>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4876 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚197915195366400 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 5494538240 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚322Î3× Ò2˜- _wandbruntime‚1936  _runtime‚1936zUª`Ç&™ÁM UIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚6039667 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17104896 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚16569482 NGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚61 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚322 B9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚5264 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 5494538240 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 125829120 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚26617676 DDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654602240 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚24153137 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚429 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚16569480 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚223 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 845178880 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 5368709120 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚4293 P8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 5494538240 D=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚60 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 4654602240 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 1738022266880 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚22609149 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 5368709120 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚6039606 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚171065660735488 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 831258624 EDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚50775106 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚209 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚22609086 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 1837723235840 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 1738022266880 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚322 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 1734440160569 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚197941748775936 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚196203684560896 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 4552982528 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚26617676 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 125829120 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 B6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚322 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚262 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚172903363327488 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚44035072 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚41348068 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 E5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5264 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚197941748775936 H@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚665 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚172903428006400 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 SFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚64678912 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 IBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚60 _runtime‚1936.314541733 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚4295 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚196783679775061 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚262 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚41948160 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17104896 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚4293 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚195049298001920 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 5494538240 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚50770811 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 4552982528 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 1738064215040 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚41948160 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 1734398812501 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 5368709120 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚50770811 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚322 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚262 _step‚591 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚24157430 K>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59053056 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚196203667456000 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚197941689722880 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚196203684560896 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 122905600 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521206rÃÂqšÆM J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚26617674 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚195049280962560 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 125829120 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚322 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚262 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚172903363327488 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚44035072 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚41348068 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 E5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5264 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚197941748775936 H@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚665 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚172903428006400 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 SFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚64678912 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 IBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚60 _runtime‚1936.314541733 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚4295 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.f“>[reed‚196783679775061 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚262 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚41948160 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17104896 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚4293 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚195049298001920 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 5494538240 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚50770811 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 4552982528 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 1738064215040 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚41948160 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 1734398812501 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 5368709120 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚50770811 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚322 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚262 _step‚591 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚24157430 K>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59053056 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚196203667456000 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚197941689722880 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚196203684560896 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 122905600 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521206 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚6039667 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17104896 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚16569482 NGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚61 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚322 B9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚5264 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 5494538240 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 125829120 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚26617676 DDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654602240 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚24153137 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚429 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚16569480 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚223 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 845178880 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 5368709120 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚4293 P8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 5494538240 D=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚60 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 4654602240 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 1738022266880 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚22609149 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 5368709120 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚6039606 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚171065660735488 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 831258624 EDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚50775106 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚209 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚22609086 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 1837723235840 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 1738022266880 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚322 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 1734440160569 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚197941748775936 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚196203684560896 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 4552982528 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚26617676 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 125829120 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 BDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚199637844913152 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚41945088 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚26852377 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 122905600 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 1753445690368 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 1749796022603 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 4517085184 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 4654602240 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚22806318 PIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚80 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚322 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 1753487635456 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17104896 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚22806400 C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚4289 IDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4876 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 1753445690368 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚46135296 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚209 UHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚66779136 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚172494252310528 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚322 BDeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4289 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 125829120 trainer/global_step‚7249 F>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚24337144 KBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚4876 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚262 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚197884416327680 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚199637844913152 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚24341431 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 125829120 KDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚60 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 122905600 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚51193810 JBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚665 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚4287 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚172494231666688 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 1853857343488 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚61478912 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚4287 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚174348155789312 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 5368709120 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 1753487635456 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚26852379 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 5368709120 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚262 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚262 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚197884399222784 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 4552982528 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚6088179 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 PCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚58387380 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚198463080360267 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚196713284337664 E8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚51189521 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚6088259 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚20643840 M@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚59049984 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚41945088 IBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚82 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚429 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚198463138747647 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 5494538240 % _timestamp‚1.7866606059996321e+09 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 5494538240 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 5368709120 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 125829120¿µÊZ?%¨¼O WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚322 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚322 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚16718139 IDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4876 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 1753445690368 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚46135296 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚209 UHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚66779136 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚172494252310528 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚322 BDeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4289 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 125829120 trainer/global_step‚7249 F>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚24337144 KBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚4876 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚262 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚197884416327680 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚199637844913152 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚24341431 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 125829120 KDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚60 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 122905600 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚51193810 JBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚665 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚4287 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚172494231666688 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 1853857343488 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚61478912 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚4287 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚174348155789312 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 5368709120 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 1753487635456 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚26852379 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 5368709120 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚262 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚262 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚197884399222784 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 4552982528 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚6088179 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 PCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚58387380 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚198463080360267 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚196713284337664 E8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚51189521 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚6088259 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚20643840 M@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚59049984 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚41945088 IBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚82 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚429 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚198463138747647 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 5494538240 % _timestamp‚1.7866606059996321e+09 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 5494538240 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 5368709120 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 125829120 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚196713301377024 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚26852379 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚24337144 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚16718141 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚26852377 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚24341431 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚197884399222784 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 4552982528 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 1853903478784 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚174348089010176 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 _step‚595 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 845178880 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 122306526 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚199637903963136 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚199637844913152 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚41945088 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚26852377 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 122905600 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 1753445690368 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 1749796022603 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 4517085184 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 4654602240 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚22806318 PIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚80 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚322 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 1753487635456 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17104896 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚22806400 C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚4289 IDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚22809354 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚4293 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚199664457372672 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 4654602240 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚196739470131200 _step‚596 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 122905600 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚199664398319616 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 4552982528 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚429 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 1750067943283 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 1753718707200 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚322 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚198489555113843 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521206 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚41948160 H@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚665 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 1750026595215 EDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚51200656 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 5494538240 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚322 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17104896 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 1753718707200 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚197910721560576 B9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚5264 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚262 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 122905600 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚174371449566208 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 5368709120 G@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚60 NGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚78 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 5494538240 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚26855930 NADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚59053056 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚4293 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚16720346 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚26855932 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚41348068 _runtime‚ 1954.69487693 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17104896 G@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚80 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 845178880 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚61478912 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 125829120 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚197910738665472 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 1753676759040 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚24340431 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚24344724 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚174371382790144 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 125829120 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚24344724 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚197910738665472 EDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59053056 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 4552982528 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚223 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 5368709120 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 1753676759040 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 4517085184 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 5494538240 DDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 >5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5264 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654602240 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚46132224 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 122306526 % _timestamp‚1.7866606061644008e+09 PDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0*]fÎË&«ÅM H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚4293 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚199664457372672 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 4654602240 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚196739470131200 _step‚596 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 122905600 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚199664398319616 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 4552982528 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚429 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 1750067943283 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 1753718707200 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚322 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚198489555113843 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521206 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚41948160 H@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚665 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 1750026595215 EDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚51200656 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 5494538240 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚322 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17104896 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 1753718707200 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚197910721560576 B9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚5264 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚262 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 122905600 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚174371449566208 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 5368709120 G@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚60 NGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚78 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 5494538240 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚26855930 NADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚59053056 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚4293 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚16720346 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚26855932 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚41348068 _runtime‚ 1954.69487693 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17104896 G@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚80 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 845178880 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚61478912 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 125829120 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚197910738665472 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 1753676759040 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚24340431 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚24344724 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚174371382790144 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 125829120 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚24344724 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚197910738665472 EDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59053056 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 4552982528 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚223 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 5368709120 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 1753676759040 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 4517085184 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 5494538240 DDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 >5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5264 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654602240 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚46132224 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 122306526 % _timestamp‚1.7866606061644008e+09 PDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚51196361 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚22809354Ôš¶Åt2¬- _wandbruntime‚1954  _runtime‚1954Øh4-o­é .loss/FRt1/assignment_loss‚0.86155795676729 /loss/FRt2/assignment_loss‚0.727892457699129 .loss/FRt1/detection_loss‚0.564666493156135 &loss/total_loss‚2.5988402055458266 epoch‚18 trainer/global_step‚7249 % _timestamp‚1.7866606061662235e+09 /loss/FRt2/detection_loss‚0.4447232979232724 _runtime‚1954.696138413 _step‚597ý­^¯t®î .loss/FRt1/assignment_loss‚0.86155795676729 /loss/FRt2/assignment_loss‚0.727892457699129 .loss/FRt1/detection_loss‚0.564666493156135 &loss/total_loss‚2.5988402055458266 epoch‚18 trainer/global_step‚7249 % _timestamp‚1.7866606061662235e+09 /loss/FRt2/detection_loss‚0.4447232979232724 _runtime‚1954.696138413 _step‚597ÕÏiÕö•¯‚(:Š ˆŽùÓá gpu.0.gpu‚61.0 gpu.0.memory‚11, gpu.0.memoryAllocated‚7.0313262939453125) gpu.0.memoryAllocatedBytes‚ 6039863296 gpu.0.temp‚33.0 gpu.0.powerWatts‚119.555' gpu.0.enforcedPowerLimitWatts‚500.0 gpu.0.powerPercent‚23.911 gpu.0.smClock‚1410 gpu.0.memoryClock‚1593$ gpu.0.correctedMemoryErrors‚1955# gpu.0.uncorrectedMemoryErrors‚0¿Så&°‚(:‚ ˆŽùÓüÖðâ network.sent‚ 9.673164e+06% disk./.usageGB‚486.99385833740234# disk.nvme0n1.out‚53995.87109375 disk.drbd2165.out‚0 disk.sda.out‚ 38.484375) proc.memory.percent‚13.98763656616211 proc.cpu.threads‚17 disk.sda1.in‚0 disk.sda2.in‚0 disk.sda2.out‚0" disk.nvme0n1.in‚43324.00390625 disk.dm-1.in‚ 18999.6328125 disk.dm-0.in‚43324.00390625 disk.sda3.in‚ 18999.6328125 disk.sda3.out‚ 38.484375 network.recv‚ 3.861314e+06 disk.sda1.out‚0 disk.dm-1.out‚ 38.484375$ disk.nvme0c0n1.in‚43324.00390625% disk.nvme0c0n1.out‚53995.87109375 disk.sda.in‚ 18999.6328125$ memory_percent‚66.64202213287354) disk./.usagePercent‚32.68256644808675 disk.drbd2165.in‚0 disk.dm-0.out‚53995.87109375) proc.memory.availableMB‚ 5465.37109375! proc.memory.rssMB‚ 2291.734375 cpu‚27.08975940929269åXÅæ2±- _wandbruntime‚1966  _runtime‚1966Ðä‚:²¼ $ _timestamp‚1.786660618191554e+09 _runtime‚ 1966.7215545 _step‚598 % lr-AdamW/pg1‚0.001086308902346142 % lr-AdamW/pg2‚0.001086308902346142 trainer/global_step‚7299Ÿ ³Ç³Á % lr-AdamW/pg1‚0.001086308902346142 % lr-AdamW/pg2‚0.001086308902346142 trainer/global_step‚7299 $ _timestamp‚1.786660618191554e+09 _runtime‚ 1966.7215545 _step‚598ÖyÁn92´- _wandbruntime‚1966  _runtime‚1966£Î+4%µ·O F>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 122905600 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4289 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚173646620950528 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 5494538240 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 1765032456192 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚27030029 BDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4876 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚199200533217280 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚16828391 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚20643840 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚322 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚200965607618560 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 831258624 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚262 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚24502246 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚6129389 IDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚200965548568576 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 4654602240 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17104896 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚41945088 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚22957780 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚199783990235055 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 5368709120 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 125829120 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚200965548568576 PCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚59049984 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 1765074401280 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚322 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 125829120 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚262 KDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚60 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 845178880 _runtime‚1966.725897995 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 1866217484288 JBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚665 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚209 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚175512771655680 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚46135296 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 5368709120 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚199200516112384 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚199200516112384 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚24506533 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚322 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚198022591053824 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 4552982528 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 125829120 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚4287 D;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚5264 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 5494538240 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚41945088 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚16828389 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 trainer/global_step‚7299 IDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 122905600 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4289 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚173646620950528 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 5494538240 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 1765032456192 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚27030029 BDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4876 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚199200533217280 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚16828391 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚20643840 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚322 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚200965607618560 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 831258624 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚262 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚24502246 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚6129389 IDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚200965548568576 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 4654602240 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17104896 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚41945088 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚22957780 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚199783990235055 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 5368709120 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 125829120 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚200965548568576 PCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚59049984 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 1765074401280 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚322 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 125829120 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚262 KDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚60 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 845178880 _runtime‚1966.725897995 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 1866217484288 JBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚665 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚209 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚175512771655680 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚46135296 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 5368709120 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚199200516112384 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚199200516112384 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚24506533 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚322 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚198022591053824 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 4552982528 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 125829120 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚4287 D;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚5264 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 5494538240 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚41945088 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚16828389 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 trainer/global_step‚7299 IDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚51543408 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚199810406601251 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚198048776847360 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 5494538240 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚16830594 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚51543408 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 1765263524864 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚24509826 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚223 NADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚59053056 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚199226855555072 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 122905600 H@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚665 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚22960779 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚322 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚20643840 NADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚58387428 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 1761629753891 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 4552982528 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚27033582 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚173669668323328 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 125829120 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚322 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚24509826 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521206 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚200992161028096 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚199226838450176 I@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚4876 _runtime‚1966.882500512 EDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚27033582 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚4293 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 122905600 G@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚71 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚22960708 NGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚69 >5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5264 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚199226838450176 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚322 K>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59053056 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚175536064911360 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 5494538240 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚198048759808000 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 125829120 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17104896 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚173669647679488 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 D=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚60 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚200992161028096 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 831258624 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚41348068 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 5368709120 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚200992101975040 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654602240 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17104896 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 1765263524864 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 1765305473024 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚48229376 E8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 5494538240 G@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚60 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 4654602240 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 4517085184 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚322 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚429 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚6130114 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 5368709120 % _timestamp‚1.7866606183522546e+09 _step‚600 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚27033580̧›Ì&¹ÆM LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚4293 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 1761588405823 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚51543408 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚199810406601251 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚198048776847360 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 5494538240 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚16830594 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚51543408 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 1765263524864 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚24509826 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚223 NADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚59053056 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚199226855555072 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 122905600 H@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚665 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚22960779 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚322 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚20643840 NADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚58387428 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 1761629753891 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 4552982528 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚27033582 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚173669668323328 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 125829120 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚322 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚24509826 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521206 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚200992161028096 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚199226838450176 I@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚4876 _runtime‚1966.882500512 EDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚27033582 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚4293 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 122905600 G@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚71 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚22960708 NGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚69 >5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5264 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚199226838450176 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚322 K>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59053056 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚175536064911360 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 5494538240 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚198048759808000 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 125829120 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17104896 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚173669647679488 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 D=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚60 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚200992161028096 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 831258624 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚41348068 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 5368709120 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚200992101975040 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654602240 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17104896 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 1765263524864 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 1765305473024 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚48229376 E8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 5494538240 G@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚60 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 4654602240 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 4517085184 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚322 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚429 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚6130114 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 5368709120 % _timestamp‚1.7866606183522546e+09 _step‚600 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚27033580 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚51539113 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 1765305473024 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚41948160 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 4552982528ØB» 2º- _wandbruntime‚1966  _runtime‚1966šH*r»ì /loss/FRt2/detection_loss‚0.4676892876206002 %loss/total_loss‚2.372644289094384 trainer/global_step‚7299 _runtime‚1966.883022178 0loss/FRt1/assignment_loss‚0.7526418848442701 0loss/FRt2/assignment_loss‚0.6113808094293093 epoch‚18 % _timestamp‚1.7866606183532114e+09 _step‚601 /loss/FRt1/detection_loss‚0.5409323072002048òHw¼ñ 0loss/FRt1/assignment_loss‚0.7526418848442701 0loss/FRt2/assignment_loss‚0.6113808094293093 epoch‚18 % _timestamp‚1.7866606183532114e+09 _step‚601 /loss/FRt1/detection_loss‚0.5409323072002048 /loss/FRt2/detection_loss‚0.4676892876206002 %loss/total_loss‚2.372644289094384 trainer/global_step‚7299 _runtime‚1966.883022178ÙËWPo2½- _wandbruntime‚1978  _runtime‚1978‹´Ýž¿ trainer/global_step‚7349 % _timestamp‚1.7866606303588455e+09 _runtime‚1978.890890359 _step‚602 % lr-AdamW/pg1‚0.001080925633157179 % lr-AdamW/pg2‚0.001080925633157179|'|ÉÊ¿Ä _runtime‚1978.890890359 _step‚602 % lr-AdamW/pg1‚0.001080925633157179 % lr-AdamW/pg2‚0.001080925633157179 trainer/global_step‚7349 % _timestamp‚1.7866606303588455e+09Ú¸ºÁl2À- _wandbruntime‚1978  _runtime‚1978¥Í.‡Á·O L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚24667348 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚6170532 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚176677495234048 PCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚58387380 KDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚60 IBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚60 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.aìû6%llocated‚16938641 PCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚59049984 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 5494538240 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 122905600 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 125829120 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17104896 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 1776619222016 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4289 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚174798989590528 _runtime‚1978.890967485 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚27207679 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 122306526 _step‚603 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚41348020 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚4287 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚202293311273984 JBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚665 G>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4876 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚200516650106880 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 1776661167104 IDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚202293252224000 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚51879314 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 1772919643819 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 5368709120 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚27207677 F>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 845178880 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚201104841722463 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚262 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚174798968946688 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚61478912 BDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 4628521206 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 1772960991839 IDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4876 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚200516650106880 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 1776661167104 IDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚202293252224000 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚51879314 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 1772919643819 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 5368709120 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚27207677 F>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 845178880 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚201104841722463 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚262 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚174798968946688 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚61478912 BDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 4628521206 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 1772960991839 IDeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4289 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚174798989590528 _runtime‚1978.890967485 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚27207679 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 122306526 _step‚603 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚41348020 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0ÛÆ842Ã- _wandbruntime‚1979  _runtime‚1979BmrÃÆ&ÄÀM T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚202319864683520 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚41948160 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚176700787110400 D6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚322 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 trainer/global_step‚7349 _step‚604 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654602240 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚46132224 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚24670635 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚23112169 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 845178880 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚174822036963328 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 831258624 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚322 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 125829120 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚51881865 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚202319864683520 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚51886160 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚41948160 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 5368709120 NADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚59053056 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚202319805630464 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚27211230 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚201131199701231 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚4293 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 1776892238848 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521206 % _timestamp‚1.7866606305198731e+09 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚24670635 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚199358049484800 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 125829120 G@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚60 D=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚60 _runtime‚ 1979.05016435 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 125829120 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚429 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚20643840 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 4517085184 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 122306526 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 5368709120 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚24674928 E5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5264 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚200542972444672 E6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚322 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17104896 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 5494538240 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 5494538240 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚23112248 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚209 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 5368709120 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚262 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚27211232 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚200542955339776 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚51886160 G@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚79 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚176700720334336 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 IBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚60 PDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59053056 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 4552982528 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 1776850290688 B9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚5264 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚24674928 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚174822016319488 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚61478912 NADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚58387428 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚201131258088659 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 1773191564499 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚6171402 NGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚77 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚41348068 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚262 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚199358066524160 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚4295 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚51881865 H@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚665 I@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚4876 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 1773150216431 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚223 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 1776892238848&—¶hí ÅÅM H@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚665 I@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚4876 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 1773150216431 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚223 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 1776892238848 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚202319864683520 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚41948160 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚176700787110400 D6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚322 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 trainer/global_step‚7349 _step‚604 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654602240 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚46132224 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚24670635 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚23112169 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 845178880 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚174822036963328 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 831258624 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚322 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 125829120 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚51881865 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚202319864683520 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚51886160 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚41948160 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 5368709120 NADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚59053056 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚202319805630464 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚Yðn Þ27211230 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚201131199701231 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚4293 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 1776892238848 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521206 % _timestamp‚1.7866606305198731e+09 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚24670635 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚199358049484800 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 125829120 G@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚60 D=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚60 _runtime‚ 1979.05016435 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 125829120 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚429 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚20643840 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 4517085184 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 122306526 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 5368709120 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚24674928 E5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5264 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚200542972444672 E6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚322 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17104896 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 5494538240 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 5494538240 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚23112248 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚209 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 5368709120 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚262 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚27211232 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚200542955339776 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚51886160 G@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚79 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚176700720334336 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 IBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚60 PDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59053056 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 4552982528 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 1776850290688 B9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚5264 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚24674928 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚174822016319488 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚61478912 NADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚58387428 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚201131258088659 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 1773191564499 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚6171402 NGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚77 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚41348068 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚262 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚199358066524160 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚4295 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚51881865Ü€_“ë2Æ- _wandbruntime‚1979  _runtime‚1979!½4oÇé trainer/global_step‚7349 $ _timestamp‚1.786660630520847e+09 _runtime‚1979.050657832 /loss/FRt2/assignment_loss‚0.648105206130983 /loss/FRt1/detection_loss‚0.5402902170385157 %loss/total_loss‚2.355587224168321 _step‚605 0loss/FRt1/assignment_loss‚0.6869428488847764 .loss/FRt2/detection_loss‚0.480248952114046 epoch‚18" °ôtÈî trainer/global_step‚7349 $ _timestamp‚1.786660630520847e+09 _runtime‚1979.050657832 /loss/FRt2/assignment_loss‚0.648105206130983 /loss/FRt1/detection_loss‚0.5402902170385157 %loss/total_loss‚2.355587224168321 _step‚605 0loss/FRt1/assignment_loss‚0.6869428488847764 .loss/FRt2/detection_loss‚0.480248952114046 epoch‚18Ý®Á±”É‚(:‰ —ŽùÓ깑á gpu.0.gpu‚62.0 gpu.0.memory‚11, gpu.0.memoryAllocated‚7.0313262939453125) gpu.0.memoryAllocatedBytes‚ 6039863296 gpu.0.temp‚35.0 gpu.0.powerWatts‚178.89' gpu.0.enforcedPowerLimitWatts‚500.0 gpu.0.powerPercent‚35.778 gpu.0.smClock‚1410 gpu.0.memoryClock‚1593$ gpu.0.correctedMemoryErrors‚1955# gpu.0.uncorrectedMemoryErrors‚0Šh!G˜Ê‚(: —ŽùÓîÏ£ã$ proc.memory.availableMB‚5465.375 network.sent‚ 9.743722e+06 disk.sda.out‚ 38.68359375 disk.sda1.out‚0 disk.sda2.out‚0& disk.nvme0n1.out‚54137.05224609375# disk.dm-0.out‚54137.05224609375 disk.sda3.in‚ 18999.6328125! proc.memory.rssMB‚ 2291.734375) disk./.usagePercent‚32.69145272560777 disk.dm-1.in‚ 18999.6328125 disk.drbd2165.in‚0 disk.dm-0.in‚43324.00390625 disk.sda3.out‚ 38.68359375 cpu‚27.090196195728915 disk.dm-1.out‚ 38.68359375 disk.drbd2165.out‚0$ disk.nvme0c0n1.in‚43324.00390625( disk.nvme0c0n1.out‚54137.05224609375) proc.memory.percent‚13.98763656616211 proc.cpu.threads‚17 network.recv‚ 3.888994e+06% disk./.usageGB‚487.12627029418945 disk.sda.in‚ 18999.6328125 disk.sda1.in‚0 disk.sda2.in‚0" disk.nvme0n1.in‚43324.00390625$ memory_percent‚66.64199829101562¼Ô“K2Ë- _wandbruntime‚1991  _runtime‚1991×gZÆÌÀ & lr-AdamW/pg1‚0.0010755211092331833 & lr-AdamW/pg2‚0.0010755211092331833 trainer/global_step‚7399 $ _timestamp‚1.786660642698042e+09 _runtime‚1991.228052488 _step‚606¥DMßËÍÅ $ _timestamp‚1.786660642698042e+09 _runtime‚1991.228052488 _step‚606 & lr-AdamW/pg1‚0.0010755211092331833 & lr-AdamW/pg2‚0.0010755211092331833 trainer/global_step‚7399Þ M8”2Î- _wandbruntime‚1991  _runtime‚1991 Ž ¼'϶O `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚201832766996480 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 1890790199296 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚322 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 125829120 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 PIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚69 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚177842148429824 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚177842083747840 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚202425693209871 _runtime‚1991.232916462 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚52222066 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 845178880 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 5368709120 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 5368709120 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚322 $ _timestamp‚1.786660642698928e+09 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚17048891 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 5494538240 IBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚60 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 125829120 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚4287 G>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4876 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 122905600 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 831258624 KBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚4876 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚322 _step‚607 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚27385329 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚27385327 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚41945088 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚200641153368064 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚201832766996480 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 4654602240 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 1890746161152 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚41945088 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 5494538240 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚262 IDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 1784481454427 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 4654602240 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚27385329 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 JBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚665 C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚4289 IBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚71 UHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚64681984 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 1784522802447 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 4552982528 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 PCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚58387380 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 4517085184 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 122905600 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 4628521206 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 122306526 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚322 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚24836737 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚24836737 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚4287 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚203621014929408 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚6211388 KDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚60 D;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚5264 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 5494538240 @7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚5264 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚175951337586688 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚61478912 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 125829120 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 M@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚59049984 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 4552982528 PCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚59049984 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚200641170407424 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 1788205987840 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚223 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚6211457 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 trainer/global_step‚7399 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚203621014929408 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚201832749891584 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 1788247932928 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4289 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚44038144 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚203620955879424 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17104896 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚17048889 IDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17104896 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚262 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚262 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 1788247932928 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚20643840 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚41348020 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚23260348 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚203620955879424 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚24832450 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 5368709120 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 E8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚52217777n´Ô^Á'лO ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚177842083747840 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚202425693209871 _runtime‚1991.232916462 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚52222066 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 845178880 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 5368709120 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 5368709120 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚322 $ _timestamp‚1.786660642698928e+09 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚17048891 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 5494538240 IBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚60 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 125829120 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚4287 G>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4876 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 122905600 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 831258624 KBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚4876 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚322 _step‚607 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚27385329 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚27385327 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚41945088 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚200641153368064 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚201832766996480 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 4654602240 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 1890746161152 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚41945088 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 5494538240 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚262 IDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 1784481454427 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 4654602240 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚27385329 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 JBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚665 C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚4289 IBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚71 UHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚64681984 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 1784522802447 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 4552982528 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 PCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚58387380 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 4517085184 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 122905600 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 4628521206 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 122306526 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚322 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚24836737 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚24836737 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚4287 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚203621014929408 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚6211388 KDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚60 D;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚5264 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 5494538240 @7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚5264 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚175951337586688 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚61478912 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 125829120 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 M@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚59049984 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 4552982528 PCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚59049984 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚200641170407424 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 1788205987840 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚223 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚6211457 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 trainer/global_step‚7399 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚203621014929408 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚201832749891584 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 1788247932928 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4289 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚44038144 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚203620955879424 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17104896 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚17048889 IDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17104896 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚262 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚262 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 1788247932928 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚20643840 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚41348020 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚23260348 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚203620955879424 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚24832450 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 5368709120 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 E8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚52217777 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚201832766996480 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 1890790199296 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚322 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 125829120 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 PIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚69 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚177842148429824߸= ·2Ñ- _wandbruntime‚1991  _runtime‚1991zv×Û ÒÁM >5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5264 NADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚59053056 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 122905600 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 1891033818112 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 4517085184 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚201859089334272 B9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚5264 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 trainer/global_step‚7399 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17104896 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚27388882 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚24840030 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚39840768 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 125829120 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚202452051188639 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚200667356200960 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚52224617 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚52228912 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚20643840 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 831258624 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚41348068 E6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚322 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0î½bì G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚52228912 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚24835737 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 4552982528 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚429 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 5494538240 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚262 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚4293 % _timestamp‚1.7866606428603313e+09 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚17051094 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚200667339161600 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚6212189 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚177865439421440 BDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 125829120 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚262 D=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚60 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚4295 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚41948160 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 1788437056512 E6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚322 PDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚23263283 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚6212257 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚175974405603328 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654602240 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚27388880 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 1788437056512 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚322 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚201859089334272 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚201859072229376 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚52224617 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚175974384959488 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚23263353 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521206 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 1784753375107 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚262 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚41948160 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 122905600 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 4552982528 I@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚4876 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 122306526 G@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚60 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚27388880 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚223 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 125829120 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚61478912 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 1784712027039 K>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59053056 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚177865378936832 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 5494538240¥B1pÌ&ÓÆM YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 1788479004672 SFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚60484608 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 5368709120 _step‚608 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚203647568338944 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚209 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚202452109576067 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 5494538240 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚24835737 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚17051096 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 845178880 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚322 PDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚23263283 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚6212257 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚175974405603328 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654602240 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚27388880 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 1788437056512 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚322 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚201859089334272 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚201859072229376 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚52224617 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚175974384959488 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚23263353 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521206 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 1784753375107 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚262 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚41948160 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 122905600 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 4552982528 I@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚4876 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 122306526 G@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚60 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚27388880 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚223 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 125829120 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚61478912 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 1784712027039 K>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59053056 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚177865378936832 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 5494538240 >5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5264 NADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚59053056 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 122905600 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 1891033818112 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 4517085184 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚201859089334272 B9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚5264 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 trainer/global_step‚7399 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17104896 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚27388882 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚24840030 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚39840768 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 125829120 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚202452051188639 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚200667356200960 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚52224617 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚52228912 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚20643840 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 831258624 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚41348068 E6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚322 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚52228912 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚24835737 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 4552982528 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚429 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 5494538240 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚262 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚4293 % _timestamp‚1.7866606428603313e+09 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚17051094 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚200667339161600 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚6212189 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚177865439421440 BDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 125829120 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚262 D=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚60 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚4295 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚41948160 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 1788437056512 EDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚27562977 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚27562979 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚27562979 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚17159141 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚322 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚203148866781184 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚177103706226688 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚44038144 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 _runtime‚2003.604922193 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 122905600 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 845178880 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚25001839 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 4654602240 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 1799792753664 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 122905600 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 4654602240 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4289 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚23411436 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚262 PCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚58387380 KDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚60 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 4552982528 PIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚70 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚177103726870528 IBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚60 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 1796043265035 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 122306526 $ _timestamp‚1.786660655070989e+09 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚429 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 1903094198272 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚17159139 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚6252295 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 125829120 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚204948659534848 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚24997552 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚61478912 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚322 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚204948718584832 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 125829120 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 125829120 PCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚59049984 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚41348020 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 5368709120 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 IDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4876 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17104896 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚203148866781184 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚322 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚203746486309899 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 5494538240 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚204948718584832 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚41945088 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚204948659534848 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚52564818 JBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚665 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚209 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚201950460084224 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚179006821068800 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚41945088 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 M@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚59049984 KBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚4876 IBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚72 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚2231ú€ò Þ»O LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚262 PCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚58387380 KDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚60 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 4552982528 PIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚70 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚177103726870528 IBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚60 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 1796043265035 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 122306526 $ _timestamp‚1.786660655070989e+09 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚429 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 1903094198272 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚17159139 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚6252295 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 125829120 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚204948659534848 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚24997552 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚61478912 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚322 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚204948718584832 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 125829120 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 125829120 PCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚59049984 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚41348020 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 5368709120 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 IDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4876 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17104896 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚203148866781184 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚322 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚203746486309899 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 5494538240 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚204948718584832 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚41945088 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚204948659534848 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚52564818 JBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚665 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚209 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚201950460084224 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚179006821068800 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚41945088 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 M@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚59049984 KBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚4876 IBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚72 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚223 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚262 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚27562977 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚203148883886080 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 831258624 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 1903050160128 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 5494538240 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 5368709120 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚322 F>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚27562977 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚27562979 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚27562979 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚17159141 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚322 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚203148866781184 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚177103706226688 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚44038144 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 _runtime‚2003.604922193 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 122905600 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 845178880 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚25001839 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 4654602240 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 1799792753664 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 122905600 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 4654602240 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4289 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚23411436ãÓw 2ß- _wandbruntime‚2003  _runtime‚2003qÎEÛÇ&àÁM [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚203175206223872 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚52567369 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚46132224 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚201976645877760 G@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚60 _runtime‚2003.764231517 >5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5264 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚27566530 B9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚5264 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚6253051 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 1796315185715 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚262 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 122905600 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚41348068 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚203175206223872 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚17161346 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚322 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚25005132 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 1800023822336 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 125829120 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 122905600 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚41948160 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 NADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚59053056 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚223 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚27566530 E6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚322 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 125829120 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 122306526 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚41948160 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚204975212941312 G@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚76 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚25000839 I@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚4876 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚6253125 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚177126753599488 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚61478912 D=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚60 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚52567369 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚4293 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 1800023822336 H@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚665 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚209 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚177126774243328 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚201976628838400 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 5494538240 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 5494538240 EDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59053056 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚179030046496768 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 5368709120 DDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521206 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 5368709120 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 5368709120 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚25005132 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚17161344 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚262 trainer/global_step‚7449 % _timestamp‚1.7866606552339165e+09 _step‚612 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 4654602240 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654602240 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17104896 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 NGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚74 SFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚66776064 NADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚58387428 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 4552982528 IBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚60 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚179030113272832 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 831258624 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚20643840 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 1796273837647 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 125829120 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚4295 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚204975271994368 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚203175189118976 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 845178880 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚322 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚262 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚322 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚203175189118976 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚52571664 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚4293 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚203772961063475 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚27566532 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚429 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚203772902676047 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚52571664 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚23414471 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚23414395Ž;!Ì&áÆM % _timestamp‚1.7866606552339165e+09 _step‚612 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 4654602240 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654602240 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17104896 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 NGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚74 SFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚66776064 NADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚58387428 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 4552982528 IBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚60 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚179030113272832 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 831258624 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚20643840 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 1796273837647 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 125829120 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚4295 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚204975271994368 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚203175189118976 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 845178880 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚322 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚262 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚322 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚203175189118976 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚52571664 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚4293 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚203772961063475 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚27566532 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚429 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚203772902676047 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚52571664 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚23414471 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚23414395 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚203175206223872 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚52567369 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚46132224 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚201976645877760 G@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚60 _runtime‚2003.764231517 >5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5264 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚27566530 B9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚5264 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚6253051 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 1796315185715 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚262 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 122905600 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚41348068 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚203175206223872 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚17161346 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚322 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚25005132 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 1800023822336 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 125829120 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 122905600 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚41948160 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 NADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚59053056 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚223 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚27566530 E6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚322 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 125829120 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 122306526 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚41948160 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚204975212941312 G@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚76 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚25000839 I@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚4876 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚6253125 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚177126753599488 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚61478912 D=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚60 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚52567369 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚4293 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 1800023822336 H@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚665 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚209 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚177126774243328 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚201976628838400 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 5494538240 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 5494538240 EDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59053056 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚179030046496768 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 5368709120 DDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521206 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 5368709120 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 5368709120 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚25005132 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚17161344 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚262 trainer/global_step‚7449ä—áâ2â- _wandbruntime‚2003  _runtime‚2003õ&ô1sãí /loss/FRt1/detection_loss‚0.5214595429294228 %loss/total_loss‚2.196006488818509 trainer/global_step‚7449 % _timestamp‚1.7866606552349355e+09 0loss/FRt1/assignment_loss‚0.6246264934780587 0loss/FRt2/assignment_loss‚0.5988787785112509 0loss/FRt2/detection_loss‚0.45104167389977645 epoch‚18 _runtime‚2003.764738945 _step‚613%¢‡xäò % _timestamp‚1.7866606552349355e+09 0loss/FRt1/assignment_loss‚0.6246264934780587 0loss/FRt2/assignment_loss‚0.5988787785112509 0loss/FRt2/detection_loss‚0.45104167389977645 epoch‚18 _runtime‚2003.764738945 _step‚613 /loss/FRt1/detection_loss‚0.5214595429294228 %loss/total_loss‚2.196006488818509 trainer/global_step‚7449åM妡å‚(:– µŽùÓÕ›…á gpu.0.gpu‚64.0 gpu.0.memory‚12, gpu.0.memoryAllocated‚7.0313262939453125) gpu.0.memoryAllocatedBytes‚ 6039863296 gpu.0.temp‚35.0 gpu.0.powerWatts‚122.542' gpu.0.enforcedPowerLimitWatts‚500.0) gpu.0.powerPercent‚24.508399999999998 gpu.0.smClock‚1410 gpu.0.memoryClock‚1593$ gpu.0.correctedMemoryErrors‚1955# gpu.0.uncorrectedMemoryErrors‚0y£¿“æ‚(:ˆ µŽùÓ‡¬Åã" disk.nvme0n1.in‚43324.00390625 disk.dm-1.out‚ 38.984375 disk.drbd2165.out‚0 network.recv‚ 3.940292e+06$ disk./.usageGB‚487.6282730102539$ disk.nvme0c0n1.in‚43324.00390625 disk.sda.in‚ 18999.6328125 disk.sda2.out‚0 disk.dm-1.in‚ 18999.6328125' proc.memory.availableMB‚ 5465.328125! proc.memory.rssMB‚ 2291.734375 disk.sda3.in‚ 18999.6328125 disk.sda1.out‚0 disk.sda2.in‚0& disk.nvme0n1.out‚54145.83251953125$ memory_percent‚66.64228439331055 proc.cpu.threads‚17# disk.dm-0.out‚54145.83251953125 disk.sda3.out‚ 38.984375 disk.sda.out‚ 38.984375 disk.drbd2165.in‚0) proc.memory.percent‚13.98763656616211 cpu‚27.09082156396295 network.sent‚ 9.900988e+06) disk./.usagePercent‚32.72514254909119 disk.dm-0.in‚43324.00390625( disk.nvme0c0n1.out‚54145.83251953125 disk.sda1.in‚04òÈ12ç- _wandbruntime‚2015  _runtime‚2015Üó¶MÇèÁ & lr-AdamW/pg1‚0.0010646496870128405 & lr-AdamW/pg2‚0.0010646496870128405 trainer/global_step‚7499 % _timestamp‚1.7866606673054607e+09 _runtime‚2015.835453119 _step‚614¿eÌéÆ _step‚614 & lr-AdamW/pg1‚0.0010646496870128405 & lr-AdamW/pg2‚0.0010646496870128405 trainer/global_step‚7499 % _timestamp‚1.7866606673054607e+09 _runtime‚2015.835453119涄O2ê- _wandbruntime‚2015  _runtime‚2015x9ÓØ ë·O IBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚60 _runtime‚2015.839869792 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17104896 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚206276422240256 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚25162654 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚178256095510528 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 4517085184 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 5494538240 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚204464983670784 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚17269389 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚205067337797307 PIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚61 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 125829120 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚429 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚52907570 D;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚5264 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 trainer/global_step‚7499 _step‚615 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚23562716 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚23562653 M@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚59049984 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚180171397700608 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚322 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚203259749761024 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 1807605075643 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4289 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚322 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚6293264 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚17269391 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚203259732721664 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚4287 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚209 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚20446500077568øQ”û0 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 5368709120 KDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚60 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 1811379519488 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 IDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 1915366872064 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 5494538240 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚25162654 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 4552982528 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17104896 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚180171462382592 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚262 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 4654602240 BDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚206276363190272 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 4552982528 JBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚665 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚178256074866688 IDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4876 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚206276363190272 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 KBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚4876 E8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚52903281 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚204464983670784 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚20643840 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚27740627 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚206276422240256 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 1915322833920 F?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚60 C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚4289 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 1811421464576 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚41945088 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚25166941 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0ßàfgÂ'ì¼O NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚4287 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚209 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚204465000775680 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 5368709120 KDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚60 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 1811379519488 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 IDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 1915366872064 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 5494538240 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚25162654 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 4552982528 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17104896 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚180171462382592 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚262 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 4654602240 BDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚206276363190272 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 4552982528 JBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚665 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚178256074866688 IDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4876 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚206276363190272 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 KBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚4876 E8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚52903281 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚204464983670784 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚20643840 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚27740627 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚206276422240256 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 1915322833920 F?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚60 C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚4289 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 1811421464576 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚41945088 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚25166941 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 IBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚60 _runtime‚2015.839869792 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17104896 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚206276422240256 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚25162654 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚178256095510528 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 4517085184 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 5494538240 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚204464983670784 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚17269389 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚205067337797307 PIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚61 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 125829120 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚429 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚52907570 D;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚5264 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 trainer/global_step‚7499 _step‚615 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚23562716 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚23562653 M@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚59049984 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚180171397700608 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚322 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚203259749761024 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 1807605075643 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4289 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚322 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚6293264 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚17269391 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚203259732721664çMÇîU2í- _wandbruntime‚2015  _runtime‚2015^ô[sÇ&îÁM ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 G@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚60 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚17271594 NGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚63 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 5368709120 D=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚60 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654602240 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 1811610588160 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 1807835648255 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 122306526 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚262 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚262 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 4552982528 H@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚665 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚25170234 I@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚4876 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚180194755471360 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚4293 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚204491306008576 EDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚23565652 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 845178880 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 >5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5264 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚27744180 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚206302975649792 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 122905600 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 DDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521206 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 125829120 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚204491323113472 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚52910121 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚6294058 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚178279142883328 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 5368709120 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 IBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚60 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚206302916596736 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚17271596 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 831258624 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 1915568552960 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚322 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17104896 B9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚5264 SFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚64678912 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚204491306008576 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚25165941 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 trainer/global_step‚7499 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚25165941 K>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59053056 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 4654602240 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17104896 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚223 G@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚65 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚52910121 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚25170234 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚23565717 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚205093754163455 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 5368709120 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 4517085184 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚322 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚262 EDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚52914416 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚4293 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚180194690792448 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 1915612588032 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 125829120 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚322 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚27744182 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚27744180¨h¹†'ïÆM REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚41948160 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚52914416 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚4293 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚180194690792448 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 1915612588032 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 125829120 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚322 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚27744182 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚27744180 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 G@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚60 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚17271594 NGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚63 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 5368709120 D=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚60 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654602240 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 1811610588160 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 1807835648255 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 122306526 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚262 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚262 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 4552982528 H@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚665 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚25170234 I@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚4876 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚180194755471360 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚4293 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚204491306008576 EDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚23565652 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 845178880 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 >5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5264 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚27744180 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚206302975649792 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 122905600 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 DDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521206 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 125829120 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚204491323113472 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚52910121 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚6294058 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚178279142883328 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 5368709120 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 IBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚60 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚206302916596736 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚17271596 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 831258624 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 1915568552960 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚322 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17104896 B9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚5264 SFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚64678912 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚204491306008576 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚25165941 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 trainer/global_step‚7499 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚25165941 K>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59053056 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 4654602240 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17104896 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚223 G@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚65 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚52910121 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚25170234 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚23565717 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚205093754163455 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 5368709120 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 4517085184 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚322 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚262 EDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 4654602240 _runtime‚2028.051001726 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 1823008230400 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚20643840 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 1927612383232 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 122905600 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚207604125895680 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 122905600 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚4287 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚181336055889920 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 4654602240 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 1822966285312 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 1819208234271 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 122306526 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 5368709120 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 125829120 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚322 KBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚4876 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚23714116 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 5368709120 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚41348020 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚41945088 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 1823008230400 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4289 D;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚5264 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚25332043 IBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚83 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 831258624 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 5494538240 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚179408464150528 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚322 _step‚619 C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚4289 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚53250322 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚61478912 BDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4876 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17104896 IDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 5368709120 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚27918277 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚204569039437824 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 trainer/global_step‚7549 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚207604066845696 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 4552982528 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚23714033 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚207604066845696 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚262 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚206388189284715 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚25332043 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚25327756 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚205781117665280 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 IBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚60 @7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚5264 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚27918279 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚181336128960512 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 5494538240 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 E8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚53246033 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚205781117665280 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚41945088 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚209 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚25327756IÓ7Â'ú¼O G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚429 PCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚58387380 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 5494538240 JBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚665 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚322 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 1822966285312 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚27918277 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚6334394 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚179408443506688 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 4628521206 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 4517085184 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 125829120 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚4287 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚17379641 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚17379639 KDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚60 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 125829120 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚207604125895680 PCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚59049984 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚205781100560384 M@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚59049984 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚205781100560384 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚6334475 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚52426752 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 5368709120 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚27918277 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚204569039437824 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 trainer/global_step‚7549 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚207604066845696 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 4552982528 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚23714033 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚207604066845696 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚262 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚206388189284715 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚25332043 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚25327756 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚205781117665280 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 IBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚60 @7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚5264 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚27918279 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚181336128960512 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 5494538240 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 E8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚53246033 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚205781117665280 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚41945088 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚209 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚25327756 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚322 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚262 F?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚60 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17104896 UHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚73070592 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 845178880 % _timestamp‚1.7866606795207627e+09 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 4552982528 PIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚81 F>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 4654602240 _runtime‚2028.051001726 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 1823008230400 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚20643840 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 1927612383232 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 122905600 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚207604125895680 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 122905600 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚4287 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚181336055889920 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 4654602240 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 1822966285312 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 1819208234271 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 122306526 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 5368709120 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 125829120 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚322 KBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚4876 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚23714116 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 5368709120 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚41348020 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚41945088 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 1823008230400 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4289 D;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚5264 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚25332043 IBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚83 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 831258624 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 5494538240 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚179408464150528 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚322 _step‚619 C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚4289 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚53250322 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚61478912 BDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4876 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17104896 IDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 % _timestamp‚1.7866606796828978e+09 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚27921832 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 4654602240 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚53252873 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚25331043 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚205807440003072 H@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚665 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚23717057 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚61478912 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 1819438806931 G@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚60 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 122905600 NGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚66 trainer/global_step‚7549 _step‚620 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 122905600 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚207630620252160 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚53252873 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚223 SFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚60484608 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 1819397458863 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚4293 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚322 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 5494538240 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚53257168 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 BDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59053056 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚322 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚39840768 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 G@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚68 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚20643840 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 125829120 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚322 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚262 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 DDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 D=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚60 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚205807422898176 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 4552982528 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚207630679305216 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚23717125 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 831258624 NADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚58387428 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 5368709120 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚205807440003072 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 125829120 >5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5264 EDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521206 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 1823239302144 I@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚4876 E6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚322 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 B9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚5264 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚429 _runtime‚2028.213123938 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚25335336 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654602240 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚41348068 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚262 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚262 PDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚53257168 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 BDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59053056 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚322 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚39840768 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 G@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚68 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚20643840 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 125829120 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚322 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚262 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 DDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 D=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚60 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚205807422898176 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 4552982528 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚207630679305216 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚23717125 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 831258624 NADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚58387428 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 5368709120 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚205807440003072 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 125829120 >5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5264 EDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521206 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 1823239302144 I@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚4876 E6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚322 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 B9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚5264 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚429 _runtime‚2028.213123938 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚25335336 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654602240 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚41348068 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚262 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚262 PDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 % _timestamp‚1.7866606796828978e+09 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚27921832 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 4654602240 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚53252873 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚25331043 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚205807440003072 H@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚665 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚23717057 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚61478912 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 1819438806931 G@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚60 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 122905600 NGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚66 trainer/global_step‚7549 _step‚620 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 122905600 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚207630620252160 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚53252873 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚223 SFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚60484608 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 1819397458863 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚4293 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚322ìtÇ2þ- _wandbruntime‚2028  _runtime‚2028Çê8sÿí 0loss/FRt1/assignment_loss‚0.8229228732638819 0loss/FRt2/assignment_loss‚0.6950123255015938 /loss/FRt2/detection_loss‚0.5167038170291122 epoch‚18 _runtime‚2028.213602362 /loss/FRt1/detection_loss‚0.5231299121678429 &loss/total_loss‚2.5577689279624307 trainer/global_step‚7549 % _timestamp‚1.7866606796838233e+09 _step‚621« ÷x€ò 0loss/FRt1/assignment_loss‚0.8229228732638819 0loss/FRt2/assignment_loss‚0.6950123255015938 /loss/FRt2/detection_loss‚0.5167038170291122 epoch‚18 _runtime‚2028.213602362 /loss/FRt1/detection_loss‚0.5231299121678429 &loss/total_loss‚2.5577689279624307 trainer/global_step‚7549 % _timestamp‚1.7866606796838233e+09 _step‚621ífíkÄ–‚(:‹ ÓŽùÓ–Ùéà gpu.0.gpu‚61.0 gpu.0.memory‚11, gpu.0.memoryAllocated‚7.0313262939453125) gpu.0.memoryAllocatedBytes‚ 6039863296 gpu.0.temp‚35.0 gpu.0.powerWatts‚123.223' gpu.0.enforcedPowerLimitWatts‚500.0 gpu.0.powerPercent‚24.6446 gpu.0.smClock‚1410 gpu.0.memoryClock‚1593$ gpu.0.correctedMemoryErrors‚1955# gpu.0.uncorrectedMemoryErrors‚0xX- ‚‚(:• ÓŽùÓ¨ãÝâ" disk.nvme0n1.in‚43348.41796875 disk.dm-1.in‚20125.69921875 proc.cpu.threads‚17$ disk.nvme0c0n1.in‚43348.41796875 disk.sda.out‚ 39.57421875 disk.sda1.in‚0 disk.sda2.in‚0& disk.nvme0n1.out‚54485.50341796875 disk.drbd2165.out‚0! proc.memory.rssMB‚ 2291.734375* disk./.usagePercent‚32.724981776359684$ disk./.usageGB‚487.6258773803711 disk.sda3.in‚20125.69921875# disk.dm-0.out‚54485.50341796875) proc.memory.percent‚13.98763656616211 disk.sda.in‚20125.69921875 disk.sda1.out‚0 disk.dm-1.out‚ 39.57421875 disk.drbd2165.in‚0 disk.dm-0.in‚43348.41796875# memory_percent‚66.6447401046753) proc.memory.availableMB‚ 5464.92578125 cpu‚27.091577732071748 network.sent‚ 1.0032745e+07 network.recv‚ 3.985555e+06 disk.sda3.out‚ 39.57421875( disk.nvme0c0n1.out‚54485.50341796875 disk.sda2.out‚0:\2ƒ- _wandbruntime‚2040  _runtime‚2040Žo%Ç„Á & lr-AdamW/pg1‚0.0010536974286448034 & lr-AdamW/pg2‚0.0010536974286448034 trainer/global_step‚7599 % _timestamp‚1.7866606916590178e+09 _runtime‚2040.189007883 _step‚622LúÞÌ…Æ _runtime‚2040.189007883 _step‚622 & lr-AdamW/pg1‚0.0010536974286448034 & lr-AdamW/pg2‚0.0010536974286448034 trainer/global_step‚7599 % _timestamp‚1.7866606916590178e+09÷Í2†- _wandbruntime‚2040  _runtime‚2040ÐNĽ'‡·O YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚207709099159503 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚262 F>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚41945088 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚6375276 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 5494538240 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚17489889 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚20643840 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 4628521206 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 125829120 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚23865167 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 1939899497472 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 5494538240 G>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4876 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 1834553051136 @7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚5264 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚205878312075264 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17104896 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚429 PCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚59049984 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚207097217449984 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚207709040772123 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚322 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 1830728696859 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 5368709120 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚28095929 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚25497145 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚180560812146688 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 IDeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4289 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚6375198 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 125829120 F?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚60 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚25492858 UHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚66779136 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 125829120 C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚4289 D;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚5264 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚23865087 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚46135296 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚207097234554880 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 122905600 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚53593074 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚4287 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚223 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚25497145 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚180560832790528 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 122306526 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚208931829551104 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚209 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚182500778423296 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 845178880 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 trainer/global_step‚7599 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚322 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚4287 PIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚78 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚262 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 4654602240 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚28095927 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚207097234554880 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 4517085184 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚262 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚207097217449984 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 1830770044879 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚41945088 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚25492858 KBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚4876 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚41348020 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚322 KDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚60 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚208931770501120 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17104896 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚61478912 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 BDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚208931770501120 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17104896 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚61478912 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 BDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚41945088 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚6375276 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 5494538240 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚17489889 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚20643840 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 4628521206 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 125829120 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚23865167 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 1939899497472 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 5494538240 G>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4876 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 1834553051136 @7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚5264 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚205878312075264 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17104896 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚429 PCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚59049984 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚207097217449984 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚207709040772123 @8DeviceStatsMonitor.on_train_batch_start/segment.º+¡¼all.peak‚322 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 1830728696859 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 5368709120 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚28095929 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚25497145 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚180560812146688 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 IDeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4289 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚6375198 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 125829120 F?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚60 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚25492858 UHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚66779136 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 125829120 C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚4289 D;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚5264 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚23865087 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚46135296 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚207097234554880 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 122905600 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚53593074 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚4287 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚223 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚25497145 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚180560832790528 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 122306526 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚208931829551104 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚209 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚182500778423296 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 845178880 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 trainer/global_step‚7599 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚322ïbzv2‰- _wandbruntime‚2040  _runtime‚2040:¯Ÿ¡Å&Š¿M H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚4293 P5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5264 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚28099482 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 122905600 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚28099482 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚182524008242176 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚25496145 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚17492094 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 1831000617539 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚53595625 DDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 5368709120 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚262 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17104896 G@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚65 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 125829120 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚4295 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚41348068 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 5494538240 IBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚60 NADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚59053056 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚23868199 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 122306526 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚322 trainer/global_step‚7599 B9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚5264 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚262 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚17492096 NADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚58387428 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 5494538240 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17104896 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚25496145 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚223 BDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654602240 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚207123556892672 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚429 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 H@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚665 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 # _timestamp‚1.78666069182185e+09 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 122905600 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚205904514908160 D=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚60 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚6376040 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚209 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 845178880 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 1830959269471 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 5368709120 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚322 K>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59053056 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚208958323907584 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚180583880163328 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 1940148722688 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚207735515525699 _step‚624 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚208958382960640 E6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚322 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 NGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚63 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 831258624 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 1940188563456 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚39840768 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 4517085184 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 1834826067968 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚20643840 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚207123556892672 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 4552982528 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 4552982528 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 1834826067968 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚53599920 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚180583859519488 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 1834784119808 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚61478912 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 G@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚60 _runtime‚2040.352163604 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚41948160 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚28099480 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚23868134 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚6376103 SFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚60484608 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚53595625 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521206 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚322 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0ñ£W<Ê&‹ÄM XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 5368709120 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚262 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17104896 G@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚65 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 125829120 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚4295 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚41348068 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 5494538240 IBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚60 NADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚59053056 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚23868199 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 122306526 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚322 trainer/global_step‚7599 B9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚5264 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚262 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚17492096 NADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚58387428 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 5494538240 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17104896 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚25496145 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚223 BDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654602240 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚207123556892672 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚429 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 H@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚665 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 # _timestamp‚1.78666069182185e+09 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 122905600 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚205904514908160 D=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚60 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚6376040 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚209 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 845178880 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 1830959269471 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 5368709120 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚322 K>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59053056 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚208958323907584 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚180583880163328 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 1940148722688 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚207735515525699 _step‚624 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚208958382960640 E6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚322 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 NGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚63 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 831258624 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 1940188563456 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚39840768 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 4517085184 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 1834826067968 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚20643840 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚207123556892672 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 4552982528 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 4552982528 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 1834826067968 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚53599920 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚180583859519488 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 1834784119808 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚61478912 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 G@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚60 _runtime‚2040.352163604 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚41948160 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚28099480 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚23868134 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚6376103 SFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚60484608 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚53595625 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521206 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚322 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚4293 P5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5264 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚28099482 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 122905600 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚28099482 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚182524008242176 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚25496145 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚17492094 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 1831000617539 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚53595625 DDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0ðþ–Ñ©2Œ- _wandbruntime‚2040  _runtime‚2040±fuqë 0loss/FRt1/assignment_loss‚0.8489151609002525 /loss/FRt1/detection_loss‚0.6061017109890549 %loss/total_loss‚2.671209894700035 epoch‚18 trainer/global_step‚7599 _step‚625 0loss/FRt2/assignment_loss‚0.7140252867106698 /loss/FRt2/detection_loss‚0.5021677361000577 $ _timestamp‚1.786660691822825e+09 _runtime‚2040.352612592I5ëAvŽð 0loss/FRt2/assignment_loss‚0.7140252867106698 /loss/FRt2/detection_loss‚0.5021677361000577 $ _timestamp‚1.786660691822825e+09 _runtime‚2040.352612592 0loss/FRt1/assignment_loss‚0.8489151609002525 /loss/FRt1/detection_loss‚0.6061017109890549 %loss/total_loss‚2.671209894700035 epoch‚18 trainer/global_step‚7599 _step‚625ñŨ¸2- _wandbruntime‚2045  _runtime‚2045”LÜÖ (particle/accuracy‚0.6408458948135376 +particle/sensitivity‚0.5954950451850891 % _timestamp‚1.7866606968511105e+09 _step‚626 (loss/val_total_loss‚2.47554976266825 .Purity/*FRt/FRt_purity‚0.47369271516799927 *jet/accuracy_1_of_1‚0.3087731897830963 +jet/accuracy_2_of_2‚0.21284663677215576 particle/accuracy_1_of_1‚1 +particle/specificity‚0.6657958626747131 0Purity/*FRt/event_purity‚0.46036407351493835 3Purity/1FRt/event_proportion‚0.6297432780265808 -Purity/2FRt/FRt_purity‚0.5600988268852234 /particle/accuracy_2_of_2‚0.6208481192588806 6validation_average_jet_accuracy‚0.3138950765132904 2Purity/*FRt/event_proportion‚0.669805645942688 Purity/0FRt/event_purity‚1 3Purity/0FRt/event_proportion‚0.3301943838596344 0Purity/1FRt/event_purity‚0.46262282133102417 *jet/accuracy_1_of_2‚0.5757536888122559 +validation_accuracy‚0.21284663677215576 epoch‚18 .Purity/1FRt/FRt_purity‚0.46262282133102417 -particle/accuracy_1_of_2‚0.88468998670578 _runtime‚2045.381160544 0Purity/2FRt/event_purity‚0.42398956418037415 2Purity/2FRt/event_proportion‚0.040062315762043 &particle/f_score‚0.540614128112793 trainer/global_step‚7599JÈþá‘Û 2Purity/*FRt/event_proportion‚0.669805645942688 Purity/0FRt/event_purity‚1 3Purity/0FRt/event_proportion‚0.3301943838596344 0Purity/1FRt/event_purity‚0.46262282133102417 *jet/accuracy_1_of_2‚0.5757536888122559 +validation_accuracy‚0.21284663677215576 epoch‚18 .Purity/1FRt/FRt_purity‚0.46262282133102417 -particle/accuracy_1_of_2‚0.88468998670578 _runtime‚2045.381160544 0Purity/2FRt/event_purity‚0.42398956418037415 2Purity/2FRt/event_proportion‚0.040062315762043 &particle/f_score‚0.540614128112793 trainer/global_step‚7599 (particle/accuracy‚0.6408458948135376 +particle/sensitivity‚0.5954950451850891 % _timestamp‚1.7866606968511105e+09 _step‚626 (loss/val_total_loss‚2.47554976266825 .Purity/*FRt/FRt_purity‚0.47369271516799927 *jet/accuracy_1_of_1‚0.3087731897830963 +jet/accuracy_2_of_2‚0.21284663677215576 particle/accuracy_1_of_1‚1 +particle/specificity‚0.6657958626747131 0Purity/*FRt/event_purity‚0.46036407351493835 3Purity/1FRt/event_proportion‚0.6297432780265808 -Purity/2FRt/FRt_purity‚0.5600988268852234 /particle/accuracy_2_of_2‚0.6208481192588806 6validation_average_jet_accuracy‚0.3138950765132904òvúç –’‚(:‹ âŽùÓ騡á gpu.0.gpu‚65.0 gpu.0.memory‚12, gpu.0.memoryAllocated‚7.0313262939453125) gpu.0.memoryAllocatedBytes‚ 6039863296 gpu.0.temp‚35.0 gpu.0.powerWatts‚159.909' gpu.0.enforcedPowerLimitWatts‚500.0 gpu.0.powerPercent‚31.9818 gpu.0.smClock‚1410 gpu.0.memoryClock‚1593$ gpu.0.correctedMemoryErrors‚1955# gpu.0.uncorrectedMemoryErrors‚0kªÿ—“‚(:Œ âŽùÓåœëâ) disk./.usagePercent‚32.72499918487201 disk.sda3.in‚20125.69921875 disk.sda.in‚20125.69921875 disk.sda1.in‚0 disk.sda2.in‚0 disk.drbd2165.out‚0* proc.memory.percent‚14.173102378845215 network.sent‚ 1.0095925e+07% disk./.usageGB‚487.62613677978516" disk.nvme0c0n1.in‚ 43351.546875 disk.sda1.out‚0 disk.dm-1.in‚20125.69921875 disk.dm-1.out‚ 39.81640625 disk.drbd2165.in‚0 disk.dm-0.in‚ 43351.546875 network.recv‚ 4.00544e+06 disk.sda2.out‚0% disk.nvme0n1.out‚54519.7216796875' proc.memory.availableMB‚ 5434.921875# proc.memory.rssMB‚ 2322.12109375 cpu‚27.102389937173932 disk.sda3.out‚ 39.81640625' disk.nvme0c0n1.out‚54519.7216796875 disk.sda.out‚ 39.81640625 disk.nvme0n1.in‚ 43351.546875" disk.dm-0.out‚54519.7216796875# memory_percent‚66.8278694152832 proc.cpu.threads‚17ÂËH…2”- _wandbruntime‚2058  _runtime‚2058†Ò Å•¿ % lr-AdamW/pg1‚0.001048191864403642 % lr-AdamW/pg2‚0.001048191864403642 trainer/global_step‚7649 % _timestamp‚1.7866607096437678e+09 _runtime‚2058.173833071 _step‚627°Õc™Ê–Ä % lr-AdamW/pg2‚0.001048191864403642 trainer/global_step‚7649 % _timestamp‚1.7866607096437678e+09 _runtime‚2058.173833071 _step‚627 % lr-AdamW/pg1‚0.001048191864403642ó¶¼ÑÏ2—- _wandbruntime‚2058  _runtime‚2058ÅØH(䘷O UHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚62584832 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 5368709120 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚41945088 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚17640755 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 1850207543296 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚25680152 PIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚70 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 122306526 @7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚5264 IBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚72 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚182012471738368 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚61478912 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚209414857723477 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚210654538147840 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 122905600 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚4287 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚183968729845248 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 4628521206 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 KDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚60 IDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4876 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚183968792430080 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 4552982528 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚209 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 IBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚60 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚24065440 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 4517085184 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 5494538240 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 5494538240 E8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚54014335 F?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚60 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚208804288659456 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚209414916110857 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚41348020 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17104896 KBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚4876 PCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚58387380 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 125829120 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 125829120 C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚4289 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 4552982528 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚24065512 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚223 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚262 trainer/global_step‚7649 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚208804271554560 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚6424687 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚41940992 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 _runtime‚2058.177858095 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚28334185 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 1850249488384 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚6424757 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 JBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚665 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚322 IDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚25680152 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 4654602240 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚210654479097856 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 122905600 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚207568518283264 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 5494538240 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚322 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚210654479097856 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4289 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚28334183 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 5368709120 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚262ìjÜÂ'™¼O WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 4517085184 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 5494538240 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 5494538240 E8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚54014335 F?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚60 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚208804288659456 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚209414916110857 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚41348020 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17104896 KBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚4876 PCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚58387380 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 125829120 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 125829120 C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚4289 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 4552982528 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚24065512 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚223 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚262 trainer/global_step‚7649 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚208804271554560 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚6424687 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚41940992 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 _runtime‚2058.177858095 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚28334185 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 1850249488384 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚6424757 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 JBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚665 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚322 IDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚25680152 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 4654602240 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚210654479097856 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 122905600 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚207568518283264 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 5494538240 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚322 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚210654479097856 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4289 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚28334183 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 5368709120 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚262 UHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚62584832 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 5368709120 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚41945088 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚17640755 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 1850207543296 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚25680152 PIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚70 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 122306526 @7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚5264 IBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚72 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚182012471738368 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚61478912 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚209414857723477 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚210654538147840 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 122905600 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚4287 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚183968729845248 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 4628521206 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 KDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚60 IDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4876 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚183968792430080 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 4552982528 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚209 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 IBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚60 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚24065440ô4H±2š- _wandbruntime‚2058  _runtime‚20581 KãÇ&›ÁM C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚17642960 NGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚66 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚429 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚209 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 5494538240 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 5368709120 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚322 G@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚68 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚207594687037440 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 5494538240 >5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5264 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚54021175 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚24068470 NADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚58387428 trainer/global_step‚7649 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚210681091557376 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654602240 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚208830593892352 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚207594704076800 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 DDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521206 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚54025470 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚183992088304128 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚54021175 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 1850438611968 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚41948160 I@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚4876 BDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚262 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 1850480560128 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 845178880 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚25687732 K>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59053056 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚210681032504320 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 4552982528 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 122905600 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 831258624 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚208830610997248 SFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚66776064 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 4517085184 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 122306526 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚223 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚6425512 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚262 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚4293 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚25683439 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 1846628400253 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 1846587052185 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚322 _runtime‚2058.336935716 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚17642958 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 1956523060736 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚25683439 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 1850438611968 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚54025470 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚28337738 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚46132224 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚28337738 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚208830593892352 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 G@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚60 NADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚59053056 H@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚665 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚6425578 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 1956569192960 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚209441332477053 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚208830610997248 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 E6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚322 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚322 _step‚629 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚25687732 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 122905600 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 125829120 P8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚209441332477053 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚208830610997248 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 E6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚322 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚322 _step‚629 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚25687732 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 122905600 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 125829120 PDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚17642960 NGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚66 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚429 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚209 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 5494538240 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 5368709120 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚322 G@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚68 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚207594687037440 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 5494538240 >5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5264 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚54021175 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚24068470 NADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚58387428 trainer/global_step‚7649 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚210681091557376 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654602240 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚208830593892352 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚207594704076800 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 DDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521206 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚54025470 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚183992088304128 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚54021175 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 1850438611968 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚41948160 I@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚4876 BDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚262 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 1850480560128 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 845178880 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚25687732 K>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59053056 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚210681032504320 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 4552982528 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 122905600 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 831258624 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚208830610997248 SFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚66776064 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 4517085184 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 122306526 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚223 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚6425512 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚262 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚4293 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚25683439 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 1846628400253 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 1846587052185 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚322 _runtime‚2058.336935716 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚17642958 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 1956523060736 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚25683439 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 1850438611968 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚54025470 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚28337738 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚46132224 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚28337738 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚208830593892352 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 G@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚60 NADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚59053056õω÷«2- _wandbruntime‚2058  _runtime‚2058›µšëqžë trainer/global_step‚7649 % _timestamp‚1.7866607098075485e+09 _runtime‚ 2058.33735097 0loss/FRt1/assignment_loss‚0.7737255312691166 0loss/FRt2/assignment_loss‚0.6317414449392542 /loss/FRt1/detection_loss‚0.5670084861688067 0loss/FRt2/detection_loss‚0.43626355781648246 epoch‚19 _step‚630 $loss/total_loss‚2.40873902019366Ö4žívŸð 0loss/FRt1/assignment_loss‚0.7737255312691166 0loss/FRt2/assignment_loss‚0.6317414449392542 /loss/FRt1/detection_loss‚0.5670084861688067 0loss/FRt2/detection_loss‚0.43626355781648246 epoch‚19 _step‚630 $loss/total_loss‚2.40873902019366 trainer/global_step‚7649 % _timestamp‚1.7866607098075485e+09 _runtime‚ 2058.33735097ö Œ• ‚(:Š ñŽùÓÑŸþà gpu.0.gpu‚37.0 gpu.0.memory‚8, gpu.0.memoryAllocated‚7.0313262939453125) gpu.0.memoryAllocatedBytes‚ 6039863296 gpu.0.temp‚33.0 gpu.0.powerWatts‚119.454' gpu.0.enforcedPowerLimitWatts‚500.0 gpu.0.powerPercent‚23.8908 gpu.0.smClock‚1410 gpu.0.memoryClock‚1593$ gpu.0.correctedMemoryErrors‚1955# gpu.0.uncorrectedMemoryErrors‚0Š»Öš¡‚(: ñŽùӜҷã* disk./.usagePercent‚32.724987152517905 disk.sda3.in‚20125.69921875" disk.nvme0c0n1.in‚ 43351.546875 disk.sda.in‚20125.69921875 disk.sda1.out‚0 disk.sda2.in‚0# memory_percent‚66.8311357498169 network.recv‚ 4.038004e+06 disk.sda3.out‚ 40.01953125' disk.nvme0c0n1.out‚54558.9169921875 disk.nvme0n1.in‚ 43351.546875 disk.drbd2165.in‚0) proc.memory.availableMB‚ 5434.38671875$ disk./.usageGB‚487.6259574890137 disk.sda.out‚ 40.01953125 disk.sda1.in‚0 disk.sda2.out‚0 disk.dm-1.in‚20125.69921875 disk.dm-0.in‚ 43351.546875# proc.memory.rssMB‚ 2322.12109375* proc.memory.percent‚14.173102378845215 network.sent‚ 1.0172175e+07% disk.nvme0n1.out‚54558.9169921875 disk.dm-1.out‚ 40.01953125 disk.drbd2165.out‚0" disk.dm-0.out‚54558.9169921875 cpu‚27.102119816025727 proc.cpu.threads‚17“z9'2¢- _wandbruntime‚2070  _runtime‚2070c!î#Æ£À & lr-AdamW/pg2‚0.0010426671478562472 trainer/global_step‚7699 $ _timestamp‚1.786660722101908e+09 _runtime‚2070.631958384 _step‚631 & lr-AdamW/pg1‚0.0010426671478562472F&ˤŠtrainer/global_step‚7699 $ _timestamp‚1.786660722101908e+09 _runtime‚2070.631958384 _step‚631 & lr-AdamW/pg1‚0.0010426671478562472 & lr-AdamW/pg2‚0.0010426671478562472÷h»S=2¥- _wandbruntime‚2070  _runtime‚2070çªþ8½'¦·O \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚208877790920704 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚41348020 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 4654602240 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17104896 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 1861836254208 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 122905600 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚211982182753280 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚25845254 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 5368709120 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 125829120 _runtime‚2070.636864741 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4289 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚183164840378368 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 122306526 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚210120388444160 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚262 KDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚60 F?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚60 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚210120388444160 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚28511835 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚4287 D;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚5264 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚429 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚6465828 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚208877807960064 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 5368709120 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚211982241803264 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚41945088 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 1968539549184 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚262 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚41945088 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 1861794309120 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 4552982528 JBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚665 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚322 C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚4289 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 125829120 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚322 M@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚59049984 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚211982241803264 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 1861794309120 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚54361376 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚209 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 5494538240 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 F>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚6465756 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚262 _step‚632 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚25845254 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 1861836254208 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 1968589878784 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚28511833 IDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚210735767598265 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚28511835 PCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚59049984 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚24216833 @7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚5264 BDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4876 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚24216759 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚50329600 IBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚60 IDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚6465756 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚262 _step‚632 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚25845254 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 1861836254208 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 1968589878784 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚28511833 IDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚210735767598265 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚28511835 PCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚59049984 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚24216833 @7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚5264 BDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4876 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚24216759 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚50329600 IBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚60 IDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚211982182753280 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚25845254 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 5368709120 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 125829120 _runtime‚2070.636864741 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4289 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚183164840378368 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 122306526 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚210120388444160 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚262 KDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚60 F?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚60 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚210120388444160 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚28511835 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚4287 D;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚5264 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚429 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚6465828 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚208877807960064 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 5368709120 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚211982241803264 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚41945088 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 1968539549184 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚262 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚41945088 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 1861794309120 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 4552982528 JBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚665 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚322 C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚4289 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 125829120øêOC2¨- _wandbruntime‚2070  _runtime‚2070/Kâ-©ÁM BDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59053056 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 4552982528 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚17753208 >5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5264 G@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚79 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚17753210 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 EDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654602240 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚183187887751168 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 5368709120 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚41948160 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 5494538240 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚54363927 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 1862067325952 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521206 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚41348068 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚262 DDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 4654602240 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚210146710781952 K>DeviceStatsMonitor.onhû³$š_train_batch_end/allocation.all.allocated‚54368222 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 5368709120 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17104896 EDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚24219769 _runtime‚2070.799915159 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚4293 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 845178880 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚212008795212800 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚20643840 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 4517085184 trainer/global_step‚7699 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚322 I@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚4876 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 1968833661440 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 1858190210861 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 5494538240 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚212008736159744 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 1862067325952 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚28515386 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚185156721412608 SFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚73067520 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚208903993753600 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 NADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚59053056 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚6466638 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 5368709120 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚262 D=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚60 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚322 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚25852834 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17104896 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚25848541 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 125829120 IBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚60 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚210146727886848 NADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚58387428 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 H@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚665 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚210762125577033 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚208903976714240 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚322 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚4295 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚28515386 NGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚77 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚54363927 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚6466561 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 4552982528 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚4293 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚322 G@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚60 % _timestamp‚1.7866607222695348e+09 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚28515388 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚210146710781952 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚210762183964461 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 125829120 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚52423680"i¸ŽÌ&ªÆM C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚322 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚25852834 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17104896 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚25848541 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 125829120 IBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚60 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚210146727886848 NADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚58387428 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 H@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚665 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚210762125577033 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚208903976714240 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚322 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚4295 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚28515386 NGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚77 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚54363927 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚6466561 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 4552982528 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚4293 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚322 G@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚60 % _timestamp‚1.7866607222695348e+09 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚28515388 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚210146710781952 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚210762183964461 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 125829120 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚52423680 BDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59053056 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 4552982528 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚17753208 >5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5264 G@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚79 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚17753210 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 EDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654602240 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚183187887751168 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 5368709120 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚41948160 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 5494538240 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚54363927 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 1862067325952 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521206 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚41348068 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚262 DDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 4654602240 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚210146710781952 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚54368222 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 5368709120 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17104896 EDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚24219769 _runtime‚2070.799915159 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚4293 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 845178880 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚212008795212800 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚20643840 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 4517085184 trainer/global_step‚7699 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚322 I@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚4876 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 1968833661440 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 1858190210861 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 5494538240 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚212008736159744 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 1862067325952 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚28515386 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚185156721412608 SFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚73067520 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚208903993753600 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 NADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚59053056 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚6466638 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 5368709120 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚262 D=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚60ùž8† 2«- _wandbruntime‚2070  _runtime‚2070M†Ô¬r¬ì $ _timestamp‚1.786660722270659e+09 0loss/FRt1/assignment_loss‚0.8528641916456203 0loss/FRt2/assignment_loss‚0.7404188174542853 /loss/FRt2/detection_loss‚0.5371926758273787 &loss/total_loss‚2.6804313863333196 epoch‚19 _runtime‚2070.800495104 _step‚634 /loss/FRt1/detection_loss‚0.5499557014060354 trainer/global_step‚7699öâ Xw­ñ 0loss/FRt1/assignment_loss‚0.8528641916456203 0loss/FRt2/assignment_loss‚0.7404188174542853 /loss/FRt2/detection_loss‚0.5371926758273787 &loss/total_loss‚2.6804313863333196 epoch‚19 _runtime‚2070.800495104 _step‚634 /loss/FRt1/detection_loss‚0.5499557014060354 trainer/global_step‚7699 $ _timestamp‚1.786660722270659e+09úC%`92®- _wandbruntime‚2082  _runtime‚2082®å›¾Æ¯À _runtime‚ 2082.88916557 _step‚635 & lr-AdamW/pg1‚0.0010371236338448386 & lr-AdamW/pg2‚0.0010371236338448386 trainer/global_step‚7749 % _timestamp‚1.7866607343591483e+09Ê“Ð+Ë°Å & lr-AdamW/pg2‚0.0010371236338448386 trainer/global_step‚7749 % _timestamp‚1.7866607343591483e+09 _runtime‚ 2082.88916557 _step‚635 & lr-AdamW/pg1‚0.0010371236338448386ûÛUR2±- _wandbruntime‚2082  _runtime‚2082½¥Ó̼'²¶O `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚211436522438656 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 122306526 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 5494538240 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 5494538240 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 _step‚636 @7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚5264 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚26014643 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚4289 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 125829120 JBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚665 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚17861253 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 845178880 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚210187097636864 M@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚59049984 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 1873381074944 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 122905600 _runtime‚2082.893880511 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚211436522438656 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚41945088 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚54704128 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚429 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 1869521448809 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 125829120 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚26014643 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚186298060236800 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17104896 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 4552982528 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚28689485 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚20643840 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 4517085184 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 5368709120 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚262 KDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚60 trainer/global_step‚7749 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 122905600 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚28689483 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚184317209018368 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 1980811374592 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚213309945458688 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 1873423020032 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 831258624 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚4287 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚24368019 IBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚62 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚210187080597504 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 5368709120 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚213309886408704 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 D;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚5264 KBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚4876 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚6506764 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 1980851218432 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 1869480100789 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚28689485 F>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 G>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4876 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚212056619085673 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚26010356 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚223 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚6506704 UHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚60487680 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 IDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚213309886408704 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚262 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚262 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚26010356 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17104896 BDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚28689483 PIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚60 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚213309945458688 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚211436505333760 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚322 PCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚58387380 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 4654602240 IDeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4289 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚184317188374528 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚61478912 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚322 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 5494538240 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚17861255 IBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚60 PCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚59049984 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚322 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚212056560698293 E8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚54699839 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚186297999749120 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 4628521206 $ _timestamp‚1.786660734360076e+09ÉF'³»O PCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚58387380 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 4654602240 IDeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4289 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚184317188374528 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚61478912 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚322 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 5494538240 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚17861255 IBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚60 PCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚59049984 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚322 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚212056560698293 E8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚54699839 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚186297999749120 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 4628521206 $ _timestamp‚1.786660734360076e+09 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚211436522438656 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 122306526 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 5494538240 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 5494538240 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 _step‚636 @7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚5264 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚26014643 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚4289 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 125829120 JBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚665 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚17861253 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 845178880 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚210187097636864 M@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚59049984 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 1873381074944 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 122905600 _runtime‚2082.893880511 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚211436522438656 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚41945088 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚54704128 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚429 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 1869521448809 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 125829120 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚26014643 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚186298060236800 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17104896 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 4552982528 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚28689485 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚20643840 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 4517085184 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 5368709120 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚262 KDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚60 trainer/global_step‚7749 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 122905600 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚28689483 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚184317209018368 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 1980811374592 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚213309945458688 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 1873423020032 ZLDeviceStatsèÃ×@Monitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 831258624 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚4287 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚24368019 IBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚62 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚210187080597504 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 5368709120 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚213309886408704 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 D;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚5264 KBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚4876 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚6506764 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 1980851218432 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 1869480100789 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚28689485 F>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 G>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4876 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚212056619085673 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚26010356 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚223 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚6506704 UHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚60487680 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 IDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚213309886408704 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚262 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚262 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚26010356 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17104896 BDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚28689483 PIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚60 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚213309945458688 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚211436505333760 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚322ü—Ç™×2´- _wandbruntime‚2083  _runtime‚2083tr<Ç&µÁM G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚54710974 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 5368709120 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚322 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚26017936 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚211462844776448 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚28693036 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚41348068 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 _step‚637 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚213336498868224 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 122905600 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17104896 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 1873654091776 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚54710974 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚54706679 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 845178880 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚184340235747328 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚41948160 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 1873612143616 NADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚59053056 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚26013643 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 831258624 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 122306526 _runtime‚2083.057086936 D6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚322 IBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚60 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚6507531 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 4552982528 I@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚4876 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚24370991 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 5368709120 K>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59053056 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚322 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 4517085184 P6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚322 >5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5264 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 4654602240 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚41948160 H@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚665 trainer/global_step‚7749 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚211462827671552 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 125829120 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚262 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚262 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚211462827671552 G@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚68 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 D=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚60 NGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚66 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚212083035451869 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚184340256391168 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚212082977064441 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 122905600 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 5368709120 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚17863460 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 1981098671104 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚54706679 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚26013643 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚209 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚186321355062272 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 NADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚58387428 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 5494538240 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 5494538240 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚28693038 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚24370923 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚223 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚61478912 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚28693038 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚210213283430400 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 5494538240 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚26017936 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚6507465 SFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚66776064 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚186321288286208 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521206 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 EDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654602240 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 4552982528 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚17863458 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚213336439815168 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 1869710673401 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚20643840 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚210213266391040 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 % _timestamp‚1.7866607345265362e+09ÞiŽgÌ&¶ÆM ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 1869752021469 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚262 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚4295 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚213336498868224 B6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚322 >5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5264 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 4654602240 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚41948160 H@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚665 trainer/global_step‚7749 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚211462827671552 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 125829120 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚262 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚262 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚211462827671552 G@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚68 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 D=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚60 NGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚66 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚212083035451869 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚184340256391168 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚212082977064441 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 122905600 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 5368709120 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚17863460 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 1981098671104 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚54706679 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚26013643 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚209 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚186321355062272 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 NADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚58387428 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 5494538240 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 5494538240 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚28693038 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚24370923 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚223 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚61478912 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚28693038 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚210213283430400 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 5494538240 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚26017936 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚6507465 SFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚66776064 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚186321288286208 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521206 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 EDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654602240 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 4552982528 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚17863458 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚213336439815168 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 1869710673401 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚20643840 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚210213266391040 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 % _timestamp‚1.7866607345265362e+09 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚54710974 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 5368709120 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚322 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚26017936 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚211462844776448 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚28693036 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚41348068 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 _step‚637 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚213336498868224 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 122905600 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17104896 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 1873654091776 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚54710974 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚54706679 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 845178880 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚184340235747328 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚41948160 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 1873612143616 NADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚59053056 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚26013643 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 831258624 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 122306526 _runtime‚2083.057086936 D6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚322 IBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚60 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚6507531 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 4552982528 I@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚4876 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚24370991 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 5368709120 K>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59053056 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚322 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 4517085184 PDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚214637590064128 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 1885009785856 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚24519682 PIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚71 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚24519609 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 IBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚73 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚17971503 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚61478912 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 G>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4876 UHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚64681984 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 1993086562304 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 _runtime‚2095.091885223 M@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚59049984 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚26179745 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚322 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚212752622223360 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚41348020 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚322 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚214637649114112 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚187462643576832 @7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚5264 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚26175458 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 1884967840768 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚429 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 1881041911397 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚4287 D;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚5264 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚187462708258816 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 % _timestamp‚1.7866607465578806e+09 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚28867135 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚212752622223360 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚211496370274304 E8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚55042591 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 4552982528 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 1885009785856 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚28867135 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 122306526 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 5494538240 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚185469577658368 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚262 _step‚640 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚55046880 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4289 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚209 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚185469557014528 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 4628521206 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚322 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚26179745 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚212752639328256 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 122905600 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 4517085184 ZLDeviceStatsMonitor.on_train_±À€_batch_start/inactive_split_bytes.large_pool.peak‚ 831258624 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 5494538240 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17104896 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚41945088 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚28867133 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚26175458 BDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 4552982528 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 4654602240 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚20643840 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚213377412185701 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 IDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 4552982528 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 4654602240 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚20643840 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚213377412185701 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 IDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚214637590064128 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 1885009785856 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚24519682 PIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚71 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚24519609 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 IBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚73 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚17971503 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚61478912 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 G>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4876 UHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚64681984 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 1993086562304 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 _runtime‚2095.091885223 M@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚59049984 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚26179745 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚322 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚212752622223360 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚41348020 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚322 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚214637649114112 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚187462643576832 @7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚5264 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚26175458 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 1884967840768 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚429 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 1881041911397 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚4287 D;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚5264 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚187462708258816 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 % _timestamp‚1.7866607465578806e+09 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚28867135 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚212752622223360 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚211496370274304 E8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚55042591 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 4552982528 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 1885009785856 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚28867135 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 122306526 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 5494538240 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚185469577658368 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚262 _step‚640 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚55046880 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4289 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚209 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚185469557014528 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 4628521206 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚322 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚26179745 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚212752639328256 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 122905600 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 4517085184 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 831258624 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 5494538240 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17104896 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚41945088 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚28867133 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚26175458 BDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 trainer/global_step‚7799 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚17973708 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚28870686 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚41348068 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 122306526 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 122905600 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚4295 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚211522556067840 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 4517085184 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 5494538240 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚28870688 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 1993333329920 BDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521206 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 5494538240 EDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59053056 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17104896 H@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚665 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚187486000299008 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 845178880 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚55049431 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 NADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚58387428 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 1881313832077 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 125829120 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚322 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚322 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚262 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 4552982528 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚55053726 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 831258624 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚213403886939277 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17104896 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚429 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚17973710 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 125829120 _runtime‚2095.252335703 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚55053726 I@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚4876 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 5368709120 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 125829120 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚24522676 $ _timestamp‚1.786660746722021e+09 _step‚641 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654602240 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚211522573107200 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 EDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 5368709120 D=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚60 NADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚59053056 P5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5264 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚214664143470592 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚322 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 5368709120 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚262 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚41948160 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 5494538240 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚26178745 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 4654602240 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚4293 G@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚69 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚6549035 NGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚67 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚209 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚26178745 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚213403828551849 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 1885198909440 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚26183038 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚41948160 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚55049431 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚185492604387328 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚322 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚223 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 4552982528ç"êÄÄÅM NADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚59053056 P5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5264 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚214664143470592 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚322 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 5368709120 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚262 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚41948160 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 5494538240 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚26178745 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 4654602240 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚4293 G@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚69 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚6549035 NGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚67 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚209 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚26178745 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚213403828551849 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 1885198909440 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚26183038 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚41948160 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚55049431 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚185492604387328 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚322 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚223 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 4552982528 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚26183038 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 trainer/global_step‚7799 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚17973708 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚28870686 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚41348068 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 122306526 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 122905600 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚4295 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚211522556067840 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 4517085184 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 5494538240 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚28870688 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 1993333329920 BDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521206 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 5494538240 EDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59053056 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17104896 H@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚665 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚187486000299008 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 845178880 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚55049431 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 NADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚58387428 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 1881313832077 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 125829120 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚322 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚322 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚262 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 4552982528 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚55053726 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 831258624 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚213403886939277 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17104896 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚429 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_Ã^¬pool.allocated‚17973710 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 125829120 _runtime‚2095.252335703 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚55053726 I@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚4876 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 5368709120 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 125829120 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚24522676 $ _timestamp‚1.786660746722021e+09 _step‚641 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654602240 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚211522573107200 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 EDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 5368709120 D=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚60J{Î2Å- _wandbruntime‚2095  _runtime‚2095q¤$¨tÆî 0loss/FRt2/assignment_loss‚0.6263193266838378 trainer/global_step‚7799 % _timestamp‚1.7866607467229576e+09 /loss/FRt1/detection_loss‚0.5399943280774483 0loss/FRt2/detection_loss‚0.49711565377311945 &loss/total_loss‚2.3654567881489204 epoch‚19 _runtime‚2095.252759282 _step‚642 0loss/FRt1/assignment_loss‚0.7020274796145148$WžyÇó epoch‚19 _runtime‚2095.252759282 _step‚642 0loss/FRt1/assignment_loss‚0.7020274796145148 0loss/FRt2/assignment_loss‚0.6263193266838378 trainer/global_step‚7799 % _timestamp‚1.7866607467229576e+09 /loss/FRt1/detection_loss‚0.5399943280774483 0loss/FRt2/detection_loss‚0.49711565377311945 &loss/total_loss‚2.3654567881489204‚ÅwÅ È‚(:• ùÓÁ¿šá gpu.0.gpu‚37.0 gpu.0.memory‚8, gpu.0.memoryAllocated‚7.0313262939453125) gpu.0.memoryAllocatedBytes‚ 6039863296 gpu.0.temp‚34.0 gpu.0.powerWatts‚120.123' gpu.0.enforcedPowerLimitWatts‚500.0) gpu.0.powerPercent‚24.024600000000003 gpu.0.smClock‚1410 gpu.0.memoryClock‚1593$ gpu.0.correctedMemoryErrors‚1955# gpu.0.uncorrectedMemoryErrors‚0&d4šÉ‚(: ùÓÙ¯éã# proc.memory.rssMB‚ 2322.12109375 cpu‚27.10254771484488 proc.cpu.threads‚17 network.recv‚ 4.085422e+06) disk./.usagePercent‚32.72405989322888$ disk./.usageGB‚487.6121406555176" disk.nvme0c0n1.in‚ 43351.546875 disk.sda.out‚ 40.34765625 disk.sda2.in‚0 network.sent‚ 1.0326982e+07# disk.dm-0.out‚54569.27294921875 disk.sda3.out‚ 40.34765625 disk.sda1.out‚0 disk.sda2.out‚0 disk.nvme0n1.in‚ 43351.546875' proc.memory.availableMB‚ 5372.703125* proc.memory.percent‚14.173102378845215 disk.sda1.in‚0 disk.dm-0.in‚ 43351.546875 disk.sda3.in‚20125.69921875 disk.sda.in‚20125.69921875& disk.nvme0n1.out‚54569.27294921875 disk.dm-1.in‚20125.69921875 disk.dm-1.out‚ 40.34765625 disk.drbd2165.out‚0( disk.nvme0c0n1.out‚54569.27294921875 disk.drbd2165.in‚0$ memory_percent‚67.20762252807617YÿÐ,2Ê- _wandbruntime‚2107  _runtime‚2107rMÆËÀ trainer/global_step‚7849 $ _timestamp‚1.786660758873636e+09 _runtime‚2107.403648404 _step‚643 & lr-AdamW/pg1‚0.0010259816388126222 & lr-AdamW/pg2‚0.0010259816388126222¤iÕ]ËÌÅ & lr-AdamW/pg1‚0.0010259816388126222 & lr-AdamW/pg2‚0.0010259816388126222 trainer/global_step‚7849 $ _timestamp‚1.786660758873636e+09 _runtime‚2107.403648404 _step‚643ƒ¢>º62Í- _wandbruntime‚2107  _runtime‚2107o ³Ì½'ηO PCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚59049984 IDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4876 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17104896 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚26340560 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚24670971 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚186621925654528 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 4552982528 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚48232448 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚322 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚262 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚322 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 F>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚4287 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 125829120 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚188627369654272 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚215965352769536 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚212805676990464 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 1892603722005 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚29044785 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚215965293719552 IBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚80 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚18081755 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚209 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚188627300777984 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4289 PIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚78 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚20643840 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 4517085184 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 1892645070025 C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚4289 E8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚55385343 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚214068739112960 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚214698322060489 KDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚60 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 1896596551680 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 F?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚60 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚215965293719552 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚55389632 D;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚5264 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚186621946298368 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 4628521206 trainer/global_step‚7849 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚214698263673109 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 IDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4876 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17104896 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚26340560 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚24670971 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚186621925654528 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 4552982528 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚48232448 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚322 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚262 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚322 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 F>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚4287 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 125829120 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚188627369654272 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚215965352769536 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚212805676990464 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 1892603722005 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚29044785 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚215965293719552 IBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚80 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚18081755 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚209 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚188627300777984 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4289 PIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚78 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚20643840 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 4517085184 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 1892645070025 C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚4289 E8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚55385343 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚214068739112960 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚214698322060489 KDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚60 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 1896596551680 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0„]IÐ2Ð- _wandbruntime‚2107  _runtime‚2107Ú`õ!˜ÑÁM A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚322 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 4654602240 I@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚4876 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 2005624676352 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚223 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚214095078555648 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚29048336 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚212831862784000 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 1892834294617 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 831258624 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚214724738426685 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚212831845744640 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 1896827623424 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚209 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚214724680039257 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 5494538240 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 _runtime‚2107.566977532 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521206 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚41348068 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚55396478 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚4293 K>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59053056 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚186644973027328 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 125829120 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 1896785675264 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17104896 IBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚60 D=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚60 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 4552982528 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚41948160 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 125829120 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 5494538240 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 5368709120 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚429 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚262 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 122905600 >5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5264 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚41948160 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚6589964 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚55396478 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚29048338 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚186644993671168 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚262 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 1896827623424 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 4517085184 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚322 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚4295 NADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚59053056 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17104896 H@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚665 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 125829120 G@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚60 % _timestamp‚1.7866607590366585e+09 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚215991906179072 B9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚5264 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 5494538240 _step‚645 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654602240 G@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚70 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚18083960 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚322 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚18083958 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚20643840 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚41937920 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚26343847 P6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚322 trainer/global_step‚7849 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚214095078555648 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 1896785675264 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚6590032 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚61478912 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 122306526 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚214095061450752 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚24673992 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚262 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚29048338 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚214095061450752 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚55392183 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚4293 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚188650593Ò„4/7703680 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 5368709120 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚29048336 EDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚24673922 SFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚62581760 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚26348140 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚215991847126016 NGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚68 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 845178880 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 D8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 5368709120 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚29048336 EDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚24673922 SFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚62581760 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚26348140 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚215991847126016 NGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚68 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 845178880 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 DDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 _runtime‚2107.566977532 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521206 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚41348068 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚55396478 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚4293 K>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59053056 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚186644973027328 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 125829120 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 1896785675264 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17104896 IBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚60 D=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚60 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 4552982528 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚41948160 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 125829120 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 5494538240 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 5368709120 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚429 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚262 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 122905600 >5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5264 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚41948160 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚6589964 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚55396478 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚29048338 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚186644993671168 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚262 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 1896827623424 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 4517085184 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚322 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚4295 NADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚59053056 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17104896 H@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚665 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 125829120 G@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚60 % _timestamp‚1.7866607590366585e+09 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚215991906179072 B9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚5264 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 5494538240 _step‚645 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654602240 G@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚70 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚18083960 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚322 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚18083958 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚20643840 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚41937920 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚26343847 P6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚322 trainer/global_step‚7849 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚214095078555648 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 1896785675264 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚6590032 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚61478912 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 122306526…)aК2Ó- _wandbruntime‚2107  _runtime‚2107l¡8}pÔê %loss/total_loss‚2.490415589269058 trainer/global_step‚7849 $ _timestamp‚1.786660759037638e+09 0loss/FRt1/assignment_loss‚0.7150220779986546 0loss/FRt2/assignment_loss‚0.6983970422460765 /loss/FRt1/detection_loss‚0.5595141543806461 .loss/FRt2/detection_loss‚0.517482314643681 epoch‚19 _runtime‚2107.567466104 _step‚646´I¨ìuÕï 0loss/FRt2/assignment_loss‚0.6983970422460765 /loss/FRt1/detection_loss‚0.5595141543806461 .loss/FRt2/detection_loss‚0.517482314643681 epoch‚19 _runtime‚2107.567466104 _step‚646 %loss/total_loss‚2.490415589269058 trainer/global_step‚7849 $ _timestamp‚1.786660759037638e+09 0loss/FRt1/assignment_loss‚0.7150220779986546†UøjÍ–Ö‚(:‹ žùÓ‡»ˆá gpu.0.gpu‚64.0 gpu.0.memory‚12, gpu.0.memoryAllocated‚7.0313262939453125) gpu.0.memoryAllocatedBytes‚ 6039863296 gpu.0.temp‚36.0 gpu.0.powerWatts‚204.796' gpu.0.enforcedPowerLimitWatts‚500.0 gpu.0.powerPercent‚40.9592 gpu.0.smClock‚1410 gpu.0.memoryClock‚1593$ gpu.0.correctedMemoryErrors‚1955# gpu.0.uncorrectedMemoryErrors‚0¡™äõ•×‚(:Š žùÓŽÚÀã( disk.nvme0c0n1.out‚54573.05517578125 disk.sda2.in‚0 disk.sda2.out‚0 disk.drbd2165.out‚0% proc.memory.availableMB‚ 5372.5625 network.sent‚ 1.0397497e+07 disk.sda.out‚ 40.54296875 disk.sda1.in‚0 disk.sda1.out‚0 disk.dm-1.out‚ 40.54296875# disk.dm-0.out‚54573.05517578125$ memory_percent‚67.20848083496094 cpu‚27.10311909236414 disk.nvme0n1.in‚ 43351.546875& disk.nvme0n1.out‚54573.05517578125 disk.dm-1.in‚20125.69921875* proc.memory.percent‚14.173102378845215 network.recv‚ 4.113078e+06) disk./.usagePercent‚32.75880600379858 disk.sda3.in‚20125.69921875 disk.sda.in‚20125.69921875 disk.drbd2165.in‚0 disk.dm-0.in‚ 43351.546875# proc.memory.rssMB‚ 2322.12109375 proc.cpu.threads‚17! disk./.usageGB‚488.1298828125 disk.sda3.out‚ 40.54296875" disk.nvme0c0n1.in‚ 43351.546875ü¯/ 2Ø- _wandbruntime‚2119  _runtime‚2119%ÒÉ ÅÙ¿ % lr-AdamW/pg1‚0.001020383873421335 % lr-AdamW/pg2‚0.001020383873421335 trainer/global_step‚7899 % _timestamp‚1.7866607710865114e+09 _runtime‚2119.616531744 _step‚647l¿÷9ÊÚÄ _runtime‚2119.616531744 _step‚647 % lr-AdamW/pg1‚0.001020383873421335 % lr-AdamW/pg2‚0.001020383873421335 trainer/global_step‚7899 % _timestamp‚1.7866607710865114e+09‡ˆØ¶C2Û- _wandbruntime‚2119  _runtime‚2119•øOe½'Ü·O O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 4628521206 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚262 PCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚59049984 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚216019115160517 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 IBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚67 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚215384856002560 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚215384856002560 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚29222435 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚187774294294528 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 4517085184 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 125829120 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚26505662 F>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚61478912 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚262 trainer/global_step‚7899 % _timestamp‚1.7866607710874488e+09 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 1908141372416 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚217292997374976 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚187774314938368 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 1904165532613 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚217293056424960 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 122905600 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 2017691630080 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚217293056424960 UHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚66779136 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 5368709120 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 PCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚58387380 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 125829120 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚26509949 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚24822007 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚29222433 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 4654602240 IDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4876 KBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚4876 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚223 PIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚65 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚55732384 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚214114949627904 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 JBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚665 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 5368709120 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚29222435 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚6630069 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚6630004 BDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚217292997374976 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚46135296 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚209 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 KDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚60 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚429 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚189791939789312 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚20643840 E8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚55728095 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 122905600 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚4287 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚322 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 5494538240 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 5494538240 IBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚60 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17104896 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚24822074 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚18192005 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚41945088 D;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚5264 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4289 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚18192003 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚41348020 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 831258624 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 2017645494784 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 5494538240 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚322 M@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚59049984 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚215384873107456 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚41945088 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚29222433 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚322 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚4287 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 4654602240 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚215384873107456 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 4552982528 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 1908183317504 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚262 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 1908141372416 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚322 _step‚648 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 4552982528 @7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚5264 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17104896 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚214114966667264 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚26505662 _runtime‚2119.617797683 C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚4289k˸ݼO PCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚58387380 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 125829120 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚26509949 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚24822007 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚29222433 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 4654602240 IDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4876 KBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚4876 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚223 PIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚65 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚55732384 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚214114949627904 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 JBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚665 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 5368709120 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚29222435 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚6630069 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚6630004 BDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚217292997374976 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚46135296 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚209 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 KDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚60 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚429 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚189791939789312 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚20643840 E8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚55728095 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 122905600 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚4287 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚322 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 5494538240 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 5494538240 IBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚60 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17104896 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚24822074 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚18192005 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚41945088 D;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚5264 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4289 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚18192003 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚41348020 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 831258624 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 2017645494784 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 5494538240 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚322 M@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚59049984 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚215384873107456 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚41945088 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚29222433 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚322 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚4287 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 4654602240 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚215384873107456 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 4552982528 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 1908183317504 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚262 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 1908141372416 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚322 _step‚648 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 4552982528 @7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚5264 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17104896 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚214114966667264 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚26505662 _runtime‚2119.617797683 C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚4289 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 4628521206 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚262 PCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚59049984 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚216019115160517 F@DeviceStatsMonitor.on_train_batch_start/reGÖðó served_bytes.all.freed‚0 IBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚67 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚215384856002560 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚215384856002560 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚29222435 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚187774294294528 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 4517085184 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 125829120 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚26505662 F>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚61478912 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚262 trainer/global_step‚7899 % _timestamp‚1.7866607710874488e+09 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 1908141372416 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚217292997374976 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚187774314938368 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 1904165532613 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚217293056424960 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 122905600 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 2017691630080 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚217293056424960 UHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚66779136 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 5368709120 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0ˆAœ2Þ- _wandbruntime‚2119  _runtime‚2119QkE“Ç&ßÁM ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚214141135421440 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 1908372441088 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 122306526 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚215411178340352 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 NGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚70 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 2017936461312 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 1904437453293 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17104896 EDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚24825020 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚429 NADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚58387428 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 5368709120 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚262 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚18194208 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚209 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚20643840 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 5494538240 P6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚322 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 4517085184 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚322 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚322 EDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521206 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 5494538240 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚262 I@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚4876 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚18194210 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚50326528 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 125829120 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚215411195445248 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚189815298772480 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 5368709120 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 5368709120 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚41948160 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 845178880 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 >5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5264 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚26513242 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚26508949 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚26508949 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚216045589914093 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 1904396105225 H@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚665 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 125829120 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 4552982528 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚223 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 122905600 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654602240 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚6630812 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚187797362311168 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚61478912 BDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59053056 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚55739230 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚55734935 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚216045531526665 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 NADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚59053056 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚322¶+aÌ&àÆM J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚29225986 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚41348068 DDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59053056 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚55739230 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚55734935 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚216045531526665 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 NADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚59053056 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚322 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚214141135421440 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 1908372441088 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 122306526 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚215411178340352 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 NGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚70 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 2017936461312 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 1904437453293 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17104896 EDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚24825020 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚429 NADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚58387428 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 5368709120 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚262 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚18194208 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚209 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚20643840 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 5494538240 P6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚322 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 4517085184 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚322 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚322 EDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521206 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 5494538240 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚262 I@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚4876 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚18194210 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚50326528 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 125829120 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚215411195445248 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚189815298772480 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 5368709120 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 5368709120 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚41948160 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 845178880 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 >5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5264 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚26513242 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚26508949 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚26508949 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚216045589914093 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 1904396105225 H@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚665 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 125829120 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 4552982528 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚223 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 122905600 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654602240 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚6630812 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚187797362311168 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚61478912 BDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚218620701030400 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚216700972892160 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 4552982528 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚218620760080384 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 4552982528 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 5368709120 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚262 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚322 trainer/global_step‚7949 _runtime‚2131.997975967 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚190956572427776 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 125829120 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚4287 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚6671184 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚188926662934528 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚41348020 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 122306526 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚216700989997056 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚56075136 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚26670764 F?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚60 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 F>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 IDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4876 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚26675051 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 2029909493248 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 1915768691241 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 5494538240 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 1919770083328 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 5368709120 @7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚5264 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚26675051 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚216700972892160 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚6671113 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚20643840 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 5368709120 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 125829120 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚218620701030400 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚18302253 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚322 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 KDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚60 JBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚665 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚215424256344064 PCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚59049984 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 831258624 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4289 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚217340025035305 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 1915727343221 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 5494538240 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚322 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 1919728138240 O@ôPiM% DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 4654602240 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 122905600 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚24973366 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚209 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚188926683578368 PCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚58387380 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚29400083 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚4287 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚48232448 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚217339966647925 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚262 C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚4289 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚216700989997056 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 4517085184 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 125829120 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚322 IBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚60 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17104896 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚429 UHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚68876288 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 2029957725696 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚215424239304704 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚223 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 122905600 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17104896 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚24973439DÀ›ÍÁ'ë»O L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 4654602240 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚41945088 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 1919728138240 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚29400085 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚18302255 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚190956641304064 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚26670764 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚41945088 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 1919770083328 IBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚73 BDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚218620701030400 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚216700972892160 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 4552982528 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚218620760080384 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 4552982528 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 5368709120 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚262 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚322 trainer/global_step‚7949 _runtime‚2131.997975967 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚190956572427776 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 125829120 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚4287 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚6671184 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚188926662934528 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚41348020 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 122306526 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚216700989997056 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚56075136 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚26670764 F?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚60 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 F>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 IDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4876 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚26675051 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 2029909493248 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 1915768691241 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 5494538240 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 1919770083328 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 5368709120 @7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚5264 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚26675051 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚216700972892160 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚6671113 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚20643840 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 5368709120 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 125829120 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚218620701030400 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚18302253 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚322 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 KDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚60 JBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚665 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚215424256344064 PCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚59049984 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 831258624 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4289 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚217340025035305 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 1915727343221 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 5494538240 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚322 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 1919728138240 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 4654602240 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 122905600 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚24973366 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚209 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚188926683578368 PCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚58387380 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚29400083 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚4287 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚48232448 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚217339966647925 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚262 C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚4289 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚216700989997056 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 4517085184 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 125829120 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚322 IBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚60 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17104896 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚429 UHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚68876288 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 2029957725696 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚215424239304704 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚223 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 122905600 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17104896 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚24973439 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚218620760080384 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚61478912 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 5494538240ŒºZ?X2ì- _wandbruntime‚2132  _runtime‚2132´ÑVÆ&íÀM C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚322 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚56081982 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚216727295229952 EDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521206 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚215450425098240 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 2030152230400 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 122306526 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 5368709120 K>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59053056 P6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚322 trainer/global_step‚7949 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚56077687 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 831258624 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 4517085184 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚262 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 NADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚58387428 >5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5264 DDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 5494538240 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 125829120 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚4293 NADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚59053056 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚209 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚61478912 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚322 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚223 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚20643840 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 5368709120 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚29403638 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 G@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚60 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚216727312334848 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚216727312334848 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17104896 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚18304458 $ _timestamp‚1.786660783629573e+09 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 1920001155072 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 845178880 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 E8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚215450442137600 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 5494538240 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚26678344 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚41948160 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚24976448 G@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚71 SFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚70970368 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 2030202556928 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚24976377 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 4552982528 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚218647254436864 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654602240 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 4654602240 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚29403636 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚190979862537728 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚216727295229952 _runtime‚2132.159918259 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 122905600 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚56081982 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚190979933508096 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 5494538240 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 125829120 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚4293 NGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚69 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚56077687 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚188949710307328 D=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚60 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 122905600 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 125829120 IBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚60 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 1920001155072 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚41948160 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 1919959206912 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 1915957915833 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚262 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚26674051 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚18304460 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚218647313489920 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚218647313489920 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚29403636 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚50326528žl¨’Ë&îÅM SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 122905600 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚56081982 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚190979933508096 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 5494538240 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 125829120 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚4293 NGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚69 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚56077687 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚188949710307328 D=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚60 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 122905600 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 125829120 IBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚60 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 1920001155072 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚41948160 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 1919959206912 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 1915957915833 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚262 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚26674051 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚18304460 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚218647313489920 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚218647313489920 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚29403636 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚50326528 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚322 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚56081982 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚216727295229952 EDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521206 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚215450425098240 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 2030152230400 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 122306526 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 5368709120 K>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59053056 P6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚322 trainer/global_step‚7949 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚56077687 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 831258624 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 4517085184 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚262 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 NADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚58387428 >5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5264 DDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 5494538240 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 125829120 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚4293 NADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚59053056 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚209 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚61478912 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚322 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚223 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚20643840 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 5368709120 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚29403638 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 G@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚60 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚216727312334848 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚216727312334848 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17104896 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚18304458 $ _timestamp‚1.786660783629573e+09 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 1920001155072 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 845178880 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 E8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚215450442137600 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 5494538240 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚26678344 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚41948160 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚24976448 G@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚71 SFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚70970368 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 2030202556928 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚24976377 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 4552982528 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚218647254436864 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654602240 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 4654602240 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚29403636 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚190979862537728 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚216727295229952 _runtime‚2132.159918259 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1Î-¦2ï- _wandbruntime‚2132  _runtime‚2132é¥vôðì /loss/FRt2/detection_loss‚0.4935791288397567 trainer/global_step‚7949 _step‚654 0loss/FRt1/assignment_loss‚0.8168512297604232 &loss/total_loss‚2.5961330389397466 epoch‚19 % _timestamp‚1.7866607836306605e+09 ø ^Ù~_runtime‚2132.160491621 0loss/FRt2/assignment_loss‚0.7571057369573356 .loss/FRt1/detection_loss‚0.528596943382231kƒ\=wññ .loss/FRt1/detection_loss‚0.528596943382231 /loss/FRt2/detection_loss‚0.4935791288397567 trainer/global_step‚7949 _step‚654 0loss/FRt1/assignment_loss‚0.8168512297604232 &loss/total_loss‚2.5961330389397466 epoch‚19 % _timestamp‚1.7866607836306605e+09 _runtime‚2132.160491621 0loss/FRt2/assignment_loss‚0.7571057369573356ŽN-³¿2ò- _wandbruntime‚2144  _runtime‚2144•ï÷Åó¿ _step‚655 % lr-AdamW/pg1‚0.001009136604535364 % lr-AdamW/pg2‚0.001009136604535364 trainer/global_step‚7999 % _timestamp‚1.7866607956513367e+09 _runtime‚2144.181339023Ó³?ÊôÄ trainer/global_step‚7999 % _timestamp‚1.7866607956513367e+09 _runtime‚2144.181339023 _step‚655 % lr-AdamW/pg1‚0.001009136604535364 % lr-AdamW/pg2‚0.001009136604535364µìÙ¥2õ- _wandbruntime‚2144  _runtime‚2144¸õ"½'ö·O PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚29577735 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚25124516 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 122306526 C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚4289 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 4552982528 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚20643840 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 5494538240 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 122905600 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚429 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚223 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚262 @7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚5264 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚41945088 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 2042172482560 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚41945088 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚219948463735808 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚29577733 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚26835866 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚26840153 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚26835866 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 122905600 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚18412505 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚218660876522713 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 2042220715008 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚322 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚190079052218368 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚322 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚218660818135333 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 1931314904064 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚18412503 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚61478912 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 4628521206 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 4517085184 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚219948404685824 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚26840153 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 125829120 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17104896 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚322 PIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚70 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 5494538240 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 5368709120 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚219948404685824 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 5494538240 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 5368709120 KDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚60 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚262 IBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚72 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚4287 M@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚59049984 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 1931356849152 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 1931356849152 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚56417888 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚192121272933376 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 1931314904064 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚4287 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚216733546020864 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚29577733 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 4654602240 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚218017106886656 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 845178880 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 125829120 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚218017089781760 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚6712013 IBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚60 % _timestamp‚1.7866607956522748e+09 _runtime‚2144.182506287 IDeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4289 KBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚4876 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚192121204057088 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 125829120 _step‚656 F>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 G>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4876 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 4654602240 D;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚5264 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚262 JBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚665 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚209 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 1927289153829 F?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚60 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚218017089781760 PCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚59049984 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚25124588 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚48232448 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚190079031574528 BDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚218660818135333 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 1931314904064 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚18412503 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚61478912 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 4628521206 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 4517085184 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚219948404685824 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚26840153 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 125829120 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17104896 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚322 PIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚70 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 5494538240 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 5368709120 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚219948404685824 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 5494538240 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 5368709120 KDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚60 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚262 IBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚72 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚4287 M@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚59049984 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 1931356849152 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 1931356849152 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚56417888 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚192121272933376 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 1931314904064 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚4287 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚216733546020864 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚29577733 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 4654602240 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚218017106886656 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 845178880 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 125829120 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚218017089781760 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚6712013 IBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚60 % _timestamp‚1.7866607956522748e+09 _runtime‚2144.182506287 IDeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4289 KBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚4876 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚192121204057088 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 125829120 _step‚656 F>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 G>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4876 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 4654602240 D;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚5264 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚262 JBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚665 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚209 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 1927289153829 F?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚60 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚218017089781760 PCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚59049984 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚25124588 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚48232448 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚190079031574528 BDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521206 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚4295 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚218687292888909 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 4517085184 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 5368709120 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚219974958092288 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚218043412119552 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 _step‚657 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 125829120 DDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 >5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5264 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 1931545972736 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚429 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚56420439 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 845178880 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚322 _runtime‚2144.347927701 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚56424734 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚190102078947328 BDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654602240 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚218043429224448 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 122905600 EDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59053056 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚218043429224448 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚25127494 NGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚64 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚192144498361344 trainer/global_step‚7999 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚6712850 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 1927519726441 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 P8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚262 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 1931587920896 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 1931545972736 H@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚665 G@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚66 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚61478912 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚20643840 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚262 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 831258624 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 2042461351936 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚29581286 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚41937920 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 125829120 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚219975017145344 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚218043412119552 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 4552982528 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚322 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚322 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚26843446 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 5368709120 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚262 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚26839153 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚41348068 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚56424734 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚29581288 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 5494538240 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 D=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚60 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 122905600 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚216759731814400 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 125829120 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 4552982528 NADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚59053056 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚18414708 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚190102099591168 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 5368709120 G@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚60 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚56420439 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚29581288 B9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚5264 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚6712786 NADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚58387428 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚4293 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 4654602240˜ØlYTúÅM NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚26843446 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 5368709120 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚262 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚26839153 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚41348068 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚56424734 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚29581288 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 5494538240 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 D=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚60 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 122905600 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚216759731814400 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 125829120 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 4552982528 NADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚59053056 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freeüf w!d‚18414708 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚190102099591168 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 5368709120 G@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚60 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚56420439 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚29581288 B9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚5264 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚6712786 NADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚58387428 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚4293 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 4654602240 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521206 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚4295 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚218687292888909 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 4517085184 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 5368709120 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚219974958092288 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚218043412119552 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 _step‚657 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 125829120 DDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 >5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5264 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 1931545972736 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚429 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚56420439 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 845178880 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚322 _runtime‚2144.347927701 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚56424734 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚190102078947328 BDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654602240 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚218043429224448 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 122905600 EDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59053056 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚218043429224448 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚25127494 NGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚64 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚192144498361344 trainer/global_step‚7999 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚6712850 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 1927519726441 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 P8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚262 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 1931587920896 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 1931545972736 H@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚665 G@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚66 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚61478912 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚20643840 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚262 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 831258624 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 2042461351936 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚29581286 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚41937920 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 125829120 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚219975017145344 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚218043412119552 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 4552982528 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚322 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚322‘Co ‘2û- _wandbruntime‚2144  _runtime‚2144á<€Prüì % _timestamp‚1.7866607958189378e+09 _runtime‚2144.348747589 _step‚658 0loss/FRt1/assignment_loss‚0.7544597840382066 0loss/FRt2/assignment_loss‚0.6404289357637488 0loss/FRt2/detection_loss‚0.48746070782661255 %loss/total_loss‚2.427793725823597 trainer/global_step‚7999 .loss/FRt1/detection_loss‚0.545444298195029 epoch‚19]q~šwýñ _runtime‚2144.348747589 _step‚658 0loss/FRt1/assignment_loss‚0.7544597840382066 0loss/FRt2/assignment_loss‚0.6404289357637488 0loss/FRt2/detection_loss‚0.48746070782661255 %loss/total_loss‚2.427793725823597 trainer/global_step‚7999 .loss/FRt1/detection_loss‚0.545444298195029 epoch‚19 % _timestamp‚1.7866607958189378e+09’iÐÈO•þ‚(:Š ¼ùÓ”œá gpu.0.gpu‚55.0 gpu.0.memory‚6, gpu.0.memoryAllocated‚7.0313262939453125) gpu.0.memoryAllocatedBytes‚ 6039863296 gpu.0.temp‚34.0 gpu.0.powerWatts‚225.717' gpu.0.enforcedPowerLimitWatts‚500.0 gpu.0.powerPercent‚45.1434 gpu.0.smClock‚1410 gpu.0.memoryClock‚1593$ gpu.0.correctedMemoryErrors‚1955# gpu.0.uncorrectedMemoryErrors‚0ãN—¦”ÿ‚(:‰ ¼ùÓ£¨îâ disk.sda2.out‚0 disk.drbd2165.out‚0 cpu‚27.104238466999558 proc.cpu.threads‚17 network.recv‚ 4.168316e+06* disk./.usagePercent‚32.767066854913445 disk.sda3.in‚ 20176.421875( disk.nvme0c0n1.out‚55356.44091796875 disk.sda.in‚ 20176.421875 disk.sda1.out‚0 disk.dm-1.out‚ 40.953125 disk.dm-0.in‚ 43365.0078125 disk.sda3.out‚ 40.953125 disk.sda2.in‚0& disk.nvme0n1.out‚55356.44091796875 disk.dm-1.in‚ 20176.421875# disk.dm-0.out‚55356.44091796875# memory_percent‚66.9884443283081) proc.memory.availableMB‚ 5408.61328125# proc.memory.rssMB‚ 2286.19921875 network.sent‚ 1.0539092e+07 disk.sda.out‚ 40.953125! disk.nvme0n1.in‚ 43365.0078125 disk.drbd2165.in‚0* proc.memory.percent‚13.953852653503418$ disk./.usageGB‚488.2529754638672# disk.nvme0c0n1.in‚ 43365.0078125 disk.sda1.in‚0ÜZ2€- _wandbruntime‚2149  _runtime‚2149¥‚‘ßÙ _runtime‚2149.372696006 0Purity/*FRt/event_purity‚0.47077515721321106 /particle/accuracy_2_of_2‚0.7069717049598694 epoch‚19 *loss/val_total_loss‚2.4416736375012738 +jet/accuracy_1_of_1‚0.31506770849227905 Purity/0FRt/event_purity‚1 3Purity/0FRt/event_proportion‚0.3301943838596344 /Purity/1FRt/event_purity‚0.4733678698539734 2Purity/2FRt/event_proportion‚0.040062315762043 particle/accuracy_1_of_1‚1 % _timestamp‚1.7866608008423686e+09 'particle/f_score‚0.5563445091247559 3Purity/1FRt/event_proportion‚0.6297432780265808 -Purity/1FRt/FRt_purity‚0.4733678698539734 0Purity/2FRt/event_purity‚0.42839327454566956 +jet/accuracy_2_of_2‚0.21338942646980286 +validation_accuracy‚0.21338942646980286 5validation_average_jet_accuracy‚0.320193886756897 2Purity/*FRt/event_proportion‚0.669805645942688 .Purity/*FRt/FRt_purity‚0.48419061303138733 *jet/accuracy_1_of_2‚0.5877611041069031 (particle/accuracy‚0.6289527416229248 +particle/specificity‚0.6142904758453369 trainer/global_step‚7999 +particle/sensitivity‚0.6555548906326294 _step‚659 -Purity/2FRt/FRt_purity‚0.5684277415275574 /particle/accuracy_1_of_2‚0.9246935844421387•=6ä‚Þ /Purity/1FRt/event_purity‚0.4733678698539734 2Purity/2FRt/event_proportion‚0.040062315762043 particle/accuracy_1_of_1‚1 % _timestamp‚1.7866608008423686e+09 'particle/f_score‚0.5563445091247559 3Purity/1FRt/event_proportion‚0.6297432780265808 -Purity/1FRt/FRt_purity‚0.4733678698539734 0Purity/2FRt/event_purity‚0.42839327454566956 +jet/accuracy_2_of_2‚0.21338942646980286 +validation_accuracy‚0.21338942646980286 5validation_average_jet_accuracy‚0.320193886756897 2Purity/*FRt/event_proportion‚0.669805645942688 .Purity/*FRt/FRt_purity‚0.48419061303138733 *jet/accuracy_1_of_2‚0.5877611041069031 (particle/accuracy‚0.6289527416229248 +particle/specificity‚0.6142904758453369 trainer/global_step‚7999 +particle/sensitivity‚0.6555548906326294 _step‚659 -Purity/2FRt/FRt_purity‚0.5684277415275574 /particle/accuracy_1_of_2‚0.9246935844421387 _runtime‚2149.372696006 0Purity/*FRt/event_purity‚0.47077515721321106 /particle/accuracy_2_of_2‚0.7069717049598694 epoch‚19 *loss/val_total_loss‚2.4416736375012738 +jet/accuracy_1_of_1‚0.31506770849227905 Purity/0FRt/event_purity‚1 3Purity/0FRt/event_proportion‚0.3301943838596344“8Yº¤¡ƒ‚(:– ËùÓ†¹‰á gpu.0.gpu‚64.0 gpu.0.memory‚12, gpu.0.memoryAllocated‚7.0313262939453125) gpu.0.memoryAllocatedBytes‚ 6039863296 gpu.0.temp‚35.0 gpu.0.powerWatts‚139.234' gpu.0.enforcedPowerLimitWatts‚500.0) gpu.0.powerPercent‚27.846799999999998 gpu.0.smClock‚1410 gpu.0.memoryClock‚1593$ gpu.0.correctedMemoryErrors‚1955# gpu.0.uncorrectedMemoryErrors‚07óÑh•„‚(:Š ËùÓÞöÒâ" disk.nvme0c0n1.in‚ 43393.671875( disk.nvme0c0n1.out‚55390.95849609375 disk.sda.out‚ 41.2734375 disk.dm-1.in‚ 21320.1171875 proc.cpu.threads‚17 network.sent‚ 1.0624528e+07 network.recv‚ 4.190104e+06* disk./.usagePercent‚32.767065830883304$ disk./.usageGB‚488.2529602050781 disk.sda1.out‚0 disk.nvme0n1.in‚ 43393.671875# disk.dm-0.out‚55390.95849609375$ memory_percent‚66.99094772338867 disk.sda3.out‚ 41.2734375 disk.sda2.in‚0& disk.nvme0n1.out‚55390.95849609375 disk.drbd2165.out‚0 disk.dm-0.in‚ 43393.671875' proc.memory.availableMB‚ 5408.203125# proc.memory.rssMB‚ 2286.19921875 disk.sda2.out‚0 disk.sda3.in‚ 21320.1171875 disk.sda.in‚ 21320.1171875 disk.sda1.in‚0 disk.dm-1.out‚ 41.2734375 disk.drbd2165.in‚0* proc.memory.percent‚13.953852653503418 cpu‚27.11343186383169O(£Ó2…- _wandbruntime‚2162  _runtime‚2162â‡z“džÁ _step‚660 & lr-AdamW/pg1‚0.0010034878234317433 & lr-AdamW/pg2‚0.0010034878234317433 trainer/global_step‚8049 % _timestamp‚1.7866608137306483e+09 _runtime‚2162.260807612Ý*‡Ì‡Æ & lr-AdamW/pg2‚0.0010034878234317433 trainer/global_step‚8049 % _timestamp‚1.7866608137306483e+09 _runtime‚2162.260807612 _step‚660 & lr-AdamW/pg1‚0.0010034878234317433”ÍÜï­2ˆ- _wandbruntime‚2162  _runtime‚2162z?>×½'‰·O WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚41348020 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚27027447 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 2058555397632 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚322 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 125829120 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 5494538240 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚27023160 _step‚661 C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚4289 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 1946969396224 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚193589288504832 M@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚59049984 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 4654602240 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17104896 IBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚73 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 5494538240 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 5368709120 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚221671113282560 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚18563369 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚20643840 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 4552982528 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚219724160991232 D;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚5264 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 1942958284563 IDeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4289 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚322 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 F?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚60 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚4287 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚25324705 PIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚71 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 4552982528 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚191530691166208 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚221671172332544 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚29815991 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚29815991 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 2058597338624 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚220366693474067 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 4517085184 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 5368709120 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚262 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚25324632 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 4654602240 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚262 @7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚5264 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17104896 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 1942916936543 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚209 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 5368709120 _runtime‚2162.264899857 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚41940992 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚262 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚219724143886336 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚18563367 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 5494538240 KDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚60 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚322 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚41945088 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 1947011341312 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚41945088 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 845178880 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 G>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4876 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 122306526 JBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚665 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚6761336 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 125829120 trainer/global_step‚8049 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚220366635086687 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚4287 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚221671172332544 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 122905600 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚56843438 UHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚62584832 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 KBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚4876 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 122905600 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚27027447 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚27023160 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚221671113282560 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 1947011341312 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚29815989 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚218423735189504 PCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚59049984 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚29815989 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚219724143886336 IDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665;Ú¥UO LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚262 @7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚5264 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17104896 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 1942916936543 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚209 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 5368709120 _runtime‚2162.264899857 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚41940992 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚262 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚219724143886336 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚18563367 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 5494538240 KDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚60 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚322 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚41945088 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 1947011341312 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚41945088 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 845178880 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 G>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4876 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 122306526 JBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚665 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚6761336 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 125829120 trainer/global_step‚8049 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚220366635086687 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚4287 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚221671172332544 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 122905600 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚56843438 UHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚62584832 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 KBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚4876 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 122905600 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚27027447 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚27023160 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚221671113282560 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 1947011341312 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚29815989 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚218423735189504 PCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚59049984 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚29815989 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚219724143886336 IDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚41348020 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚27027447 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 2058555397632 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚322 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 125829120 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 5494538240 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚27023160 _step‚661 C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚4289 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 1946969396224 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚193589288504832 M@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚59049984 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 4654602240 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17104896 IBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚73 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 5494538240 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 5368709120 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.fdðiÓ reed‚221671113282560 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚18563369 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚20643840 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 4552982528 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚219724160991232 D;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚5264 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 1942958284563 IDeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4289 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚322 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 F?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚60 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚4287 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚25324705 PIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚71 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 4552982528 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚191530691166208 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚221671172332544 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚29815991 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚29815991 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 2058597338624 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚220366693474067 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 4517085184 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 5368709120 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚262 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚25324632 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 4654602240 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0•¹«vç2‹- _wandbruntime‚2162  _runtime‚21625~¾îÇ&ŒÁM D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚29819544 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 NADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚58387428 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 5368709120 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 125829120 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 K>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59053056 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚4293 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚41348068 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚322 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚27030740 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚429 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚6762158 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 1943188857223 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚262 G@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚60 I@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚4876 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 125829120 DDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚262 _step‚662 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚27026447 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521206 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚218449920983040 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 H@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚665 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚223 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 122905600 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚322 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚4295 >5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5264 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚219750483329024 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚219750466224128 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 1947242413056 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚46132224 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 1943147509155 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 122306526 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 5494538240 IBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚60 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 1947200464896 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚56845989 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚191553717895168 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚18565572 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 4517085184 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚4293 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 4552982528 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚18565574 SFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚66776064 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 2058800748032 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚41948160 NGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚73 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚193612518643200 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 125829120 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚25327732 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚61478912 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 BDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚25327657 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 831258624 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 _runtime‚2162.430565806 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 122905600 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚219750483329024 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17104896 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 4552982528 B9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚5264 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚322 EDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚56850284 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚209 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚322 G@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚75 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚191553738539008 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚221697725742080 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚41948160 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 2058846880256 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚56845989 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17104896 NADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚59053056 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654602240 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚221697725742080 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚6762085 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 5494538240 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚29819542 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚219750466224128ÏÌ&ÆM G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚56845989 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚191553717895168 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚18565572 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 4517085184 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚4293 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 4552982528 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚18565574 SFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚66776064 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 2058800748032 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚41948160 NGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚73 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚193612518643200 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 125829120 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚25327732 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚61478912 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 BDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚25327657 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 831258624 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 _runtime‚2162.430565806 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 122905600 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚219750483329024 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17104896 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 4552982528 B9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚5264 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚322 EDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚56850284 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚209 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚322 G@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚75 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚191553738539008 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚221697725742080 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚41948160 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 2058846880256 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚56845989 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17104896 NADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚59053056 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654602240 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚221697725742080 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚6762085 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 5494538240 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚29819542 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚219750466224128 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚29819544 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 NADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚58387428 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 5368709120 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 125829120 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 K>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59053056 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚4293 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚41348068 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚322 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚27030740 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚429 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚6762158 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 1943188857223 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚262 G@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚60 I@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚4876 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 125829120 DDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚262 _step‚662 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚27026447 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521206 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚218449920983040 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 H@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚665 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚223 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 122905600 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚322 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚4295 >5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5264 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚219750483329024 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚219750466224128 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 1947242413056 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚46132224 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 1943147509155 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 122306526 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 5494538240 IBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚60 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 1947200464896–%2Ý82Ž- _wandbruntime‚2162  _runtime‚2162üTtî 0loss/FRt1/assignment_loss‚0.7049755301101746 /loss/FRt1/detection_loss‚0.5358067033871299 trainer/global_step‚8049 _runtime‚2162.431085607 _step‚663 0loss/FRt2/assignment_loss‚0.7118429148059067 0loss/FRt2/detection_loss‚0.49080004314222986 &loss/total_loss‚2.4434251914454412 epoch‚20 % _timestamp‚1.7866608139013095e+09trJLyó % _timestamp‚1.7866608139013095e+09 0loss/FRt1/assignment_loss‚0.7049755301101746 /loss/FRt1/detection_loss‚0.5358067033871299 trainer/global_step‚8049 _runtime‚2162.431085607 _step‚663 0loss/FRt2/assignment_loss‚0.7118429148059067 0loss/FRt2/detection_loss‚0.49080004314222986 &loss/total_loss‚2.4434251914454412 epoch‚20—‚Ènð2‘- _wandbruntime‚2174  _runtime‚2174“HrÆ’À trainer/global_step‚8099 $ _timestamp‚1.786660826395429e+09 _runtime‚2174.925430797 _step‚664 & lr-AdamW/pg1‚0.0009978227612789045 & lr-AdamW/pg2‚0.0009978227612789045/ۣ˓Š& lr-AdamW/pg1‚0.0009978227612789045 & lr-AdamW/pg2‚0.0009978227612789045 trainer/global_step‚8099 $ _timestamp‚1.786660826395429e+09 _runtime‚2174.925430797 _step‚664˜QÅ/2”- _wandbruntime‚2174  _runtime‚2174EȆ"•·O L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚27188262 F?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚60 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 4552982528 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17104896 IBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚77 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚429 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚41945088 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚57186190 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚29993639 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚6802336 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 5494538240 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 5368709120 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17104896 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚221040260775936 JBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚665 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚27188262 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚262 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚262 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚222998875987968 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚194753946856960 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 125829120 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚322 F>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 trainer/global_step‚8099 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 4628521206 @7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚5264 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 1954478747151 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚221040277880832 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚20643840 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚221687544961475 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 125829120 M@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚59049984 PCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚59049984 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚194753888466432 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚192683059806208 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 5494538240 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 4654602240 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 845178880 PCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚58387380 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚221687486574095 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚41348020 IDeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4289 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚25475953 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚6802411 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 4517085184 IDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4876 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 4552982528 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚18673617 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚322 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚219733024866304 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚4287 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 2070849304064 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 5494538240 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 1958556162048 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚29993641 UHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚58390528 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚61478912 BDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚222998875987968 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 1958556162048 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚25476030 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 125829120 IBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚60 KBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚4876 TGDeviceStatsMonitor.on_train_batch_start/allocation.smal¯#>Ù½l_pool.allocated‚27192549 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚209 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 1954520095171 C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚4289 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚4287 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 2070887050752 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚37746688 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 5368709120 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚222998816937984 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚221040260775936 D;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚5264 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚322 KDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚60 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚29993641nShÂ'–¼O @7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚5264 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 1954478747151 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚221040277880832 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚20643840 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚221687544961475 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 125829120 M@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚59049984 PCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚59049984 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚194753888466432 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚192683059806208 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 5494538240 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 4654602240 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 845178880 PCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚58387380 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚221687486574095 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚41348020 IDeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4289 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚25475953 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚6802411 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 4517085184 IDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4876 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 4552982528 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚18673617 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚322 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚219733024866304 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚4287 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 2070849304064 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 5494538240 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 1958556162048 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚29993641 UHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚58390528 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚61478912 BDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚222998875987968 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 1958556162048 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚25476030 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 125829120 IBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚60 KBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚4876 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚27192549 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚209 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 1954520095171 C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚4289 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚4287 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 2070887050752 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚37746688 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 5368709120 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚222998816937984 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚221040260775936 D;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚5264 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚322 KDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚60 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚29993641 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚27188262 F?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚60 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 4552982528 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17104896 IBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚77 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚429 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚41945088 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚57186190 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚29993639 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚6802336 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 5494538240 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 5368709120 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17104896 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚221040260775936 JBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚665 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚27188262 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚262 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚262 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚222998875987968 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚194753946856960 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 125829120 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚322 F>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 trainer/global_step‚8099 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 4628521206™P¹¶¡—‚(:– ÚùÓ¹«¨á gpu.0.gpu‚62.0 gpu.0.memory‚11, gpu.0.memoryAllocated‚7.0313262939453125) gpu.0.memoryAllocatedBytes‚ 6039863296 gpu.0.temp‚35.0 gpu.0.powerWatts‚220.896' gpu.0.enforcedPowerLimitWatts‚500.0) gpu.0.powerPercent‚44.179199999999994 gpu.0.smClock‚1410 gpu.0.memoryClock‚1593$ gpu.0.correctedMemoryErrors‚1955# gpu.0.uncorrectedMemoryErrors‚0¤Cb–˜‚(:‹ ÚùÓ়ã# disk.dm-0.out‚55429.37353515625 cpu‚27.113093804005295 disk.sda1.out‚0) disk./.usagePercent‚32.76693577905591$ disk./.usageGB‚488.2510223388672( disk.nvme0c0n1.out‚55429.37353515625 disk.sda.out‚ 41.6171875 disk.sda1.in‚0 disk.nvme0n1.in‚ 43393.671875 disk.dm-1.out‚ 41.6171875 network.recv‚ 4.217258e+06 disk.sda2.in‚0 disk.sda2.out‚0 disk.drbd2165.in‚0 disk.drbd2165.out‚0$ memory_percent‚66.98994636535645( proc.memory.availableMB‚ 5408.3671875# proc.memory.rssMB‚ 2286.19921875 disk.sda3.out‚ 41.6171875" disk.nvme0c0n1.in‚ 43393.671875 disk.sda.in‚ 21320.1171875 disk.dm-1.in‚ 21320.1171875* proc.memory.percent‚13.953852653503418 proc.cpu.threads‚17 network.sent‚ 1.0675439e+07 disk.sda3.in‚ 21320.1171875& disk.nvme0n1.out‚55429.37353515625 disk.dm-0.in‚ 43393.671875L®éC2™- _wandbruntime‚2175  _runtime‚2175¯6a;Ç&šÁM REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚41948160 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 1958829178880 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚219759210659840 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 NADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚58387428 IBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚60 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 845178880 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚57188741 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚29997194 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 >5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5264 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚4293 G@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚78 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 831258624 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 122905600 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚18675822 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚4293 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚192706107179008 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚192706086535168 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚322 % _timestamp‚1.7866608265595858e+09 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚221066600218624 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 122306526 G@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚60 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚27195842 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚322 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 D=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚60 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 SFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚75164672 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 _runtime‚2175.089859108 EDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚57193036 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚223 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 1954750667831 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 5368709120 I@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚4876 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚209 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 125829120 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚223025429397504 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 2071143416320 BDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654602240 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚4295 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚29997192 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17104896 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚223025370344448 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚41948160 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚29997192 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚18675824 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚54520832 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚219759193620480 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚322 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚221066600218624 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 4552982528 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚194777250595328 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚221713961327671 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 5368709120 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚262 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 122905600 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚221066583113728 B9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚5264 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 2071088895488 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 K>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59053056 PDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚25478968 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 1958787230720 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 1958787230720 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚322 trainer/global_step‚8099 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚57193036 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 1958829178880 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚6803146 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521206 _step‚666 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 125829120 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 4552982528 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚25479046 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚429 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 4517085184 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 1954709319763 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚27191549 DDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 4654602240 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚223025429397504 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 5494538240 H@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚665 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚41348068 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚262 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚262 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚221066583113728 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚27195842 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0d½Åj &›ÆM XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚194777175430656 NADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚59053056 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17104896 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚6803222 NGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚76 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚61478912 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚221713902940243 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 5494538240 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚57193036 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚223 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 1954750667831 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 5368709120 I@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚4876 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚209 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 125829120 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚223025429397504 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 2071143416320 BDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654602240 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚4295 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚29997192 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17104896 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚223025370344448 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚41948160 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚29997192 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚18675824 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚54520832 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚219759193620480 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚322 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚221066600218624 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 4552982528 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚194777250595328 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚221713961327671 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 5368709120 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚262 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 122905600 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚221066583113728 B9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚5264 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 2071088895488 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 K>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59053056 PDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚25478968 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 1958787230720 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 1958787230720 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚322 trainer/global_step‚8099 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚57193036 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 1958829178880 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚6803146 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521206 _step‚666 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 125829120 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 4552982528 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚25479046 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚429 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 4517085184 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 1954709319763 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚27191549 DDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 4654602240 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚223025429397504 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 5494538240 H@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚665 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚41348068 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚262 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚262 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚221066583113728 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚27195842 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚41948160 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 1958829178880 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚219759210659840 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 NADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚58387428 IBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚60 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 845178880 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚57188741 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚29997194 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 >5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5264 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚4293 G@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚78 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 831258624 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 122905600 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚18675822 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚4293 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚192706107179008 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚192706086535168 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚322 % _timestamp‚1.7866608265595858e+09 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚221066600218624 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 122306526 G@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚60 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚27195842 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚322 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 D=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚60 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 SFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚75164672 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 _runtime‚2175.089859108 EDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 122905600 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚57528942 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚221042314543104 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 845178880 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 5368709120 E8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚57524653 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 D;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚5264 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚6843352 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 4552982528 PCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚59049984 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17104896 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 1970142927872 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4289 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚27357651 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 _runtime‚2187.012875207 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 125829120 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17104896 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚222356394770432 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚25627287 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚25627219 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 2083178393600 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚262 M@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚59049984 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 1970184872960 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚193835428446208 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚224326520593408 JBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚665 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚30171291 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 4654602240 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚193835407802368 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 125829120 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚262 KBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚4876 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚222356394770432 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 4552982528 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 122905600 KDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚60 C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚4289 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚30171291 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚224326520593408 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚195918606839808 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚41348020 trainer/global_step‚8149 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚4287 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚27353364 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 4628521206 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 5368709120 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 1970184872960 IBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚68 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 831258624 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 5494538240 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚222356377665536 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 F?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚60 $ _timestamp‚1.786660838479356e+09 @7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚5264 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚6843418 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚209 _step‚669 BDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4876 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚223008396448883 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 4517085184 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 1966040557759 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 122306526 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚262 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚4287 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 4654602240 UHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚62584832 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 1966081905779 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0µ®TÁ'¤»O IBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚68 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 831258624 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 5494538240 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚222356377665536 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 F?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚60 $ _timestamp‚1.786660838479356e+09 @7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚5264 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚6843418 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚209 _step‚669 BDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4876 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚223008396448883 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 4517085184 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 1966040557759 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 122306526 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚262 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚4287 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 4654602240 UHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚62584832 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 1966081905779 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 IDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 122905600 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚57528942 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚221042314543104 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 845178880 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 5368709120 E8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚57524653 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 D;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚5264 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚6843352 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 4552982528 PCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚59049984 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17104896 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 1970142927872 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4289 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚27357651 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 _runtime‚2187.012875207 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 125829120 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17104896 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚222356394770432 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚25627287 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚25627219 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 2083178393600 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚262 M@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚59049984 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 1970184872960 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚193835428446208 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚224326520593408 JBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚665 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚30171291 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 4654602240 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚193835407802368 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 125829120 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚262 KBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚4876 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚222356394770432 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 4552982528 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 122905600 KDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚60 C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚4289 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚30171291 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚224326520593408 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚195918606839808 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚41348020 trainer/global_step‚8149 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚4287 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚27353364 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 4628521206 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 5368709120 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 1970184872960Z“/2¥- _wandbruntime‚2187  _runtime‚2187‘¬€Ç&¦ÁM % _timestamp‚1.7866608386423774e+09 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚57531493 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚18786072 NGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚66 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚195941899043840 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 1966271130371 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 K>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59053056 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚57535788 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 845178880 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚30174844 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17104896 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚6844159 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 125829120 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚262 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 4552982528 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚6844225 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚221068483297280 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚224353133052928 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚223 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚27356651 H@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚665 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚27360944 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚195941834364928 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 2083423224832 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚322 G@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚60 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚322 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚221068500336640 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚41348068 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 5494538240 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 125829120 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 5368709120 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 125829120 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚224353073999872 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521206 E5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5264 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚27356651 _runtime‚2187.172658994 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚41948160 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚30174842 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 831258624 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚25630299 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 DDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚25630231 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 5494538240 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 trainer/global_step‚8149 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚222382700003328 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚44035072 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 4517085184 BDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654602240 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17104896 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 1970373996544 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚57531493 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚209 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚4295 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚30174842 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 122905600 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚322 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 4552982528 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚4293 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚262 _step‚670 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚30174844 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 5494538240 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 5368709120 D=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚60 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 4654602240 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚429 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚193858475819008 NADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚58387428 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚224353133052928 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚322 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚222382700003328 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚41948160 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 122905600 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚222382717108224 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 1966312478439 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 1970415944704 B9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚5264 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚262 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 1970415944704 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚18786074 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 5368709120‚?Üo§ÆM WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚222382700003328 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚41948160 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 122905600 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚222382717108224 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 1966312478439 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 1970415944704 B9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚5264 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚262 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 1970415944704 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚18786074 ;4DeviceSta ~´¾]#tsMonitor.on_train_batch_end/max_split_size‚-1 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 5368709120 % _timestamp‚1.7866608386423774e+09 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚57531493 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚18786072 NGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚66 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚195941899043840 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 1966271130371 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 K>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59053056 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚57535788 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 845178880 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚30174844 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17104896 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚6844159 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 125829120 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚262 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 4552982528 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚6844225 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚221068483297280 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚224353133052928 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚223 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚27356651 H@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚665 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚27360944 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚195941834364928 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 2083423224832 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚322 G@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚60 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚322 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚221068500336640 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚41348068 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 5494538240 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 125829120 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 5368709120 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 125829120 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚224353073999872 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521206 E5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5264 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚27356651 _runtime‚2187.172658994 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚41948160 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚30174842 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 831258624 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚25630299 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 DDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚25630231 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 5494538240 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 trainer/global_step‚8149 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚222382700003328 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚44035072 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 4517085184 BDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654602240 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17104896 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 1970373996544 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚57531493 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚209 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚4295 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚30174842 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 122905600 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚322 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 4552982528 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚4293 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚262 _step‚670 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚30174844 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 5494538240 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 5368709120 D=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚60 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 4654602240 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚429 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚193858475819008 NADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚58387428 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚224353133052928 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚322žØgMQ2¨- _wandbruntime‚2187  _runtime‚2187‚™(ës©í 0loss/FRt2/assignment_loss‚0.5647653943664532 /loss/FRt2/detection_loss‚0.4916419977436762 &loss/total_loss‚2.2338227066516474 trainer/global_step‚8149 0loss/FRt1/assignment_loss‚0.6671091663535301 /loss/FRt1/detection_loss‚0.5103061481879879 epoch‚20 % _timestamp‚1.7866608386433651e+09 _runtime‚2187.173163175 _step‚671'xªò _runtime‚2187.173163175 _step‚671 0loss/FRt2/assignment_loss‚0.5647653943664532 /loss/FRt2/detection_loss‚0.4916419977436762 &loss/total_loss‚2.2338227066516474 trainer/global_step‚8149 0loss/FRt1/assignment_loss‚0.6671091663535301 /loss/FRt1/detection_loss‚0.5103061481879879 epoch‚20 % _timestamp‚1.7866608386433651e+09Ÿ5å T–«‚(:‹ éùÓ¯Êôà gpu.0.gpu‚63.0 gpu.0.memory‚12, gpu.0.memoryAllocated‚7.0313262939453125) gpu.0.memoryAllocatedBytes‚ 6039863296 gpu.0.temp‚34.0 gpu.0.powerWatts‚119.454' gpu.0.enforcedPowerLimitWatts‚500.0 gpu.0.powerPercent‚23.8908 gpu.0.smClock‚1410 gpu.0.memoryClock‚1593$ gpu.0.correctedMemoryErrors‚1955# gpu.0.uncorrectedMemoryErrors‚0AGjŽ¬‚(:ƒ éùÓˆÙîâ network.sent‚ 1.0761835e+07) disk./.usagePercent‚32.76705943069495" disk.nvme0c0n1.in‚ 43393.671875$ disk.nvme0c0n1.out‚ 55433.5703125 disk.dm-1.in‚ 21320.1171875 disk.drbd2165.in‚0 proc.cpu.threads‚17 disk.sda3.in‚ 21320.1171875 disk.sda3.out‚ 41.75390625 disk.sda.in‚ 21320.1171875 disk.sda1.in‚0 disk.nvme0n1.in‚ 43393.671875" disk.nvme0n1.out‚ 55433.5703125 disk.dm-1.out‚ 41.75390625$ disk./.usageGB‚488.2528648376465 disk.sda2.out‚0 disk.dm-0.in‚ 43393.671875 disk.dm-0.out‚ 55433.5703125$ memory_percent‚66.98949337005615) proc.memory.availableMB‚ 5408.44140625 cpu‚27.113133239484235 network.recv‚ 4.240353e+06 disk.sda.out‚ 41.75390625 disk.sda1.out‚0 disk.sda2.in‚0 disk.drbd2165.out‚0# proc.memory.rssMB‚ 2286.19921875* proc.memory.percent‚13.953852653503418û*gö2­- _wandbruntime‚2199  _runtime‚2199žE²[Ç®Á & lr-AdamW/pg1‚0.0009864452502733273 & lr-AdamW/pg2‚0.0009864452502733273 trainer/global_step‚8199 % _timestamp‚1.7866608508569095e+09 _runtime‚2199.386939281 _step‚672ÿ×jÌ¯Æ % _timestamp‚1.7866608508569095e+09 _runtime‚2199.386939281 _step‚672 & lr-AdamW/pg1‚0.0009864452502733273 & lr-AdamW/pg2‚0.0009864452502733273 trainer/global_step‚8199 ”2°- _wandbruntime‚2199  _runtime‚2199Ì. ½'±·O @7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚5264 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚41945088 IDeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4289 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚30348939 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚27518466 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚224329189548911 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 1977643716387 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚225654283298816 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 1977602368367 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚322 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚262 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚262 IBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚60 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚30348941 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚222351604219904 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚25778482 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 4654602240 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 122905600 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 831258624 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚4287 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚262 E8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚57867405 G>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4876 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚18894117 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 5494538240 _step‚673 F>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 D;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚5264 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚194987776442368 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 2095455727104 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 5494538240 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚197083252813312 C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚4289 M@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚59049984 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚30348941 _runtime‚2199.391891568 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 1981729693696 PCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚59049984 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 5368709120 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚223672494555136 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚61478912 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 IBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚73 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚6884436 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17104896 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 4628521206 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 122306526 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚322 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 4552982528 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚197083188131328 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚20643840 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 125829120 UHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚64681984 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚322 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 122905600 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚209 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚322 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚25778555 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚27522753 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 4654602240 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 1981771638784 KBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚4876 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚6884365 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚194987797086208 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 trainer/global_step‚8199 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚223 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 845178880 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚41348020 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚30348939 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚57871694 BDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚225654224248832 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 1981729693696 PIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚71 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚429 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 4517085184 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚44038144 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 1981771638784 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚225654224248832 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚223672494555136dHæÂ'²¼O M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚25778482 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 4654602240 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 122905600 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 831258624 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚4287 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚262 E8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚57867405 G>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4876 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚18894117 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 5494538240 _step‚673 F>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 D;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚5264 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚194987776442368 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 2095455727104 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 5494538240 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚197083252813312 C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚4289 M@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚59049984 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚30348941 _runtime‚2199.391891568 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 1981729693696 PCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚59049984 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 5368709120 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚223672494555136 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚61478912 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 IBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚73 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚6884436 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17104896 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 4628521206 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 122306526 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚322 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 4552982528 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚197083188131328 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚20643840 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 125829120 UHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚64681984 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚322 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 122905600 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚209 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚322 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚25778555 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚27522753 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 4654602240 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 1981771638784 KBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚4876 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚6884365 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚194987797086208 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 trainer/global_step‚8199 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚223 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 845178880 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚41348020 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚30348939 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚57871694 BDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚225654224248832 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 1981729693696 PIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚71 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚429 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 4517085184 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚44038144 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 1981771638784 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚225654224248832 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚223672494555136 @7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚5264 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚41945088 IDeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4289 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚30348939 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚27518466 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚224329189548911 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 1977643716387 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚225654283298816 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 1977602368367 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚322 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚262 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚262 IBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚60 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚30348941 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚222351604219904 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0¡pu Z2³- _wandbruntime‚2199  _runtime‚2199F"{X´ÁM ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚20643840 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 1977874289047 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 5494538240 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚262 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚262 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚223698816892928 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚30352494 (•ù:o$QEDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚6885161 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 122306526 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 E6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚322 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 5368709120 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654602240 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 845178880 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚223698833997824 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚30352492 _runtime‚2199.552572003 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 4654602240 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 1982002710528 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 4552982528 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚197106545541632 BDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521206 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 125829120 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚57878540 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚222377790013440 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 125829120 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚262 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚27521753 G@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚60 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚57874245 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚429 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚224355664302487 % _timestamp‚1.7866608510223083e+09 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚18896324 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚57878540 I@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚4876 G@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚72 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 5494538240 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 125829120 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚30352492 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚224355605915059 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚322 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 122905600 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚27526046 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 1977832940979 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚322 _step‚674 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 1982002710528 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚195010844459008 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 831258624 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 NADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚59053056 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚322 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚4293 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚197106476668416 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚61478912 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 4517085184 >5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5264 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17104896 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚225680836708352 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚48229376 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚57874245 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 1981960762368 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 122905600 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 trainer/global_step‚8199 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚18896322 PDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚223698833997824 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 4552982528 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 NADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚58387428 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 5368709120 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚25781483 K>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59053056 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚223698816892928 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚4295 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚27526046 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚4293 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚225680836708352 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚209 IBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚60 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚225680777655296 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 »%šÌ&µÆM QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 4552982528 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚197106545541632 BDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521206 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 125829120 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚57878540 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚222377790013440 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 125829120 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚262 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚27521753 G@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚60 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚57874245 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚429 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚224355664302487 % _timestamp‚1.7866608510223083e+09 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚18896324 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚57878540 I@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚4876 G@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚72 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 5494538240 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 125829120 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚30352492 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚224355605915059 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚322 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 122905600 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚27526046 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 1977832940979 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚322 _step‚674 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 1982002710528 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚195010844459008 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 831258624 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 NADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚59053056 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚322 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚4293 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚197106476668416 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚61478912 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 4517085184 >5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5264 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17104896 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚225680836708352 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚48229376 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚57874245 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 1981960762368 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 122905600 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 trainer/global_step‚8199 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚18896322 PDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚223698833997824 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 4552982528 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 NADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚58387428 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 5368709120 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚25781483 K>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59053056 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚223698816892928 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚4295 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚27526046 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚4293 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚225680836708352 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚209 IBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚60 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚225680777655296 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚20643840 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 1977874289047 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 5494538240 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚262 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚262 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚223698816892928 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚30352494 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚6885161 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 122306526 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 E6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚322 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 5368709120 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654602240 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 845178880 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚223698833997824 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚30352492 _runtime‚2199.552572003 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 4654602240 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 1982002710528 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0¢ì즅2¶- _wandbruntime‚2199  _runtime‚2199ƒX¥‚r·ì 0loss/FRt1/assignment_loss‚0.6724795871102843 0loss/FRt2/assignment_loss‚0.5745028448724565 &loss/total_loss‚2.2560032301454918 epoch‚20 trainer/global_step‚8199 _step‚675 /loss/FRt1/detection_loss‚0.5545724042541436 /loss/FRt2/detection_loss‚0.4544483939086076 $ _timestamp‚1.786660851023339e+09 _runtime‚2199.553140336¥„.w¸ñ /loss/FRt1/detection_loss‚0.5545724042541436 /loss/FRt2/detection_loss‚0.4544483939086076 $ _timestamp‚1.786660851023339e+09 _runtime‚2199.553140336 0loss/FRt1/assignment_loss‚0.6724795871102843 0loss/FRt2/assignment_loss‚0.5745028448724565 &loss/total_loss‚2.2560032301454918 epoch‚20 trainer/global_step‚8199 _step‚675£›,Ï‚–¹‚(:‹ øùÓé–¤á gpu.0.gpu‚63.0 gpu.0.memory‚12, gpu.0.memoryAllocated‚7.0313262939453125) gpu.0.memoryAllocatedBytes‚ 6039863296 gpu.0.temp‚34.0 gpu.0.powerWatts‚223.327' gpu.0.enforcedPowerLimitWatts‚500.0 gpu.0.powerPercent‚44.6654 gpu.0.smClock‚1410 gpu.0.memoryClock‚1593$ gpu.0.correctedMemoryErrors‚1955# gpu.0.uncorrectedMemoryErrors‚0“Ï+x”º‚(:‰ øùÓ§ñêâ cpu‚27.113380515549764 disk.sda3.in‚ 21320.1171875 disk.sda3.out‚ 42.0234375% disk.nvme0c0n1.out‚55440.00390625 disk.sda.out‚ 42.0234375 disk.sda1.in‚0# disk.nvme0n1.out‚55440.00390625 disk.dm-0.in‚43393.77734375# proc.memory.rssMB‚ 2286.19921875 network.recv‚ 4.267852e+06) disk./.usagePercent‚32.76699952493194 disk.sda.in‚ 21320.1171875 disk.sda2.in‚0" disk.nvme0n1.in‚43393.77734375 disk.dm-1.out‚ 42.0234375 disk.drbd2165.in‚0 network.sent‚ 1.0832222e+07$ disk./.usageGB‚488.2519721984863$ disk.nvme0c0n1.in‚43393.77734375 disk.drbd2165.out‚0$ memory_percent‚66.99216365814209) proc.memory.availableMB‚ 5408.00390625 proc.cpu.threads‚17 disk.sda1.out‚0 disk.sda2.out‚0 disk.dm-1.in‚ 21320.1171875 disk.dm-0.out‚55440.00390625* proc.memory.percent‚13.953852653503418‰Öÿ¾2»- _wandbruntime‚2211  _runtime‚2211ã[6Êļ¾ " _timestamp‚1.7866608632068e+09 _runtime‚2211.736904647 _step‚676 & lr-AdamW/pg1‚0.0009807335321768604 & lr-AdamW/pg2‚0.0009807335321768604 trainer/global_step‚8249å¶}ɽà _step‚676 & lr-AdamW/pg1‚0.0009807335321768604 & lr-AdamW/pg2‚0.0009807335321768604 trainer/global_step‚8249 " _timestamp‚1.7866608632068e+09 _runtime‚2211.736904647¤OTa2¾- _wandbruntime‚2211  _runtime‚2211­ñyk½'¿·O RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 122905600 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚41945088 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 1989164178975 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚58214446 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚19004367 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 5494538240 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚322 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17104896 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 1993358404608 D;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚5264 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚209 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚20643840 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 F?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚60 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 1993316459520 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 5494538240 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚27687855 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 845178880 trainer/global_step‚8249 M@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚59049984 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚198247836798976 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 125829120 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚198247907772416 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚262 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚226981927904256 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚27683568 KBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚4876 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚225650099423699 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚226981927904256 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚30526589 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 2107742046208 BDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4876 IBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚74 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚50329600 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚41348020 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 5494538240 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚262 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 % _timestamp‚1.7866608632078588e+09 E8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚58210157 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 4654602240 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 1993358404608 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚30526591 PIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚72 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚223660893896704 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 1989205526995 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 125829120 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚27687855 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚4287 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚224988628549632 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚225650041036319 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 _runtime‚2211.741896168 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 4552982528 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 125829120 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚322 F>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4289 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚6925373 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚196140165726208 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚223660876857344 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚226981986954240 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚224988611444736 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚41945088 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 122905600 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 831258624 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚322 IDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4289 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_poþš×_K%ol.freed‚6925373 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚196140165726208 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚223660876857344 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚226981986954240 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚224988611444736 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚41945088 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 122905600 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 831258624 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚322 IDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚262 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚226981927904256 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚27683568 KBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚4876 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚225650099423699 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚226981927904256 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚30526589 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 2107742046208 BDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4876 IBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚74 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚50329600 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚41348020 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 5494538240 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚262 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 % _timestamp‚1.7866608632078588e+09 E8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚58210157 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 4654602240 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 1993358404608 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚30526591 PIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚72 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚223660893896704 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 1989205526995 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 125829120 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚27687855 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚4287 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚224988628549632 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚225650041036319¥ú§2Á- _wandbruntime‚2211  _runtime‚2211ô€Æ&ÂÀM I@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚4876 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 1989394751587 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 B6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚322 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚262 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚225014933782528 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚52423680 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 122306526 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚58216997 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚25932728 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚262 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚30530144 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 831258624 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚225676457402467 E8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 >5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5264 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚41348068 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚322 trainer/global_step‚8249 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 125829120 D6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚322 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 1989436099655 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚27691148 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚227008540363776 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 5494538240 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚58221292 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 1993547528192 H@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚665 G@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚77 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 5494538240 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17104896 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚25932805 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚223 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚322 _step‚678 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚19006574 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚6926156 SFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚73067520 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚225676515789895 IBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚60 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚227008540363776 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚225014950887424 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚262 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 G@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚60 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚6926231 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚196163192455168 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 _runtime‚2211.901301287 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 4654602240 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 1993547528192 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚58221292 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚61478912 NADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚58387428 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 5494538240 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 5368709120 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚30530142 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 D=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚60 K>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59053056 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 122905600 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521206 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚223687079690240 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚223687062650880 $ _timestamp‚1.786660863371021e+09 NADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚59053056 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚19006572 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 125829120 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚225014933782528 B9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚5264 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚209 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚198271128678400 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 2107988646912 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚30530142 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚225014950887424 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 4552982528 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 4552982528 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 122905600 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 5368709120 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚4295 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚4293 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654602240 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17104896 EDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚58221292 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚61478912 NADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚58387428 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 5494538240 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 5368709120 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚30530142 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 D=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚60 K>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59053056 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 122905600 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521206 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚223687079690240 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚223687062650880 $ _timestamp‚1.786660863371021e+09 NADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚59053056 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚19006572 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 125829120 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚225014933782528 B9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚5264 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚209 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚198271128678400 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 2107988646912 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚30530142 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚225014950887424 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 4552982528 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 4552982528 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 122905600 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 5368709120 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚4295 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚4293 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654602240 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17104896 E6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚322 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚262 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚225014933782528 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚52423680 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 122306526 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚58216997 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚25932728 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚262 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚30530144 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 831258624 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚225676457402467 E8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 >5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5264 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚41348068 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚322 trainer/global_step‚8249 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 125829120 D6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚322 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 1989436099655 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚27691148 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚227008540363776 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 5494538240 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚58221292 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 1993547528192 H@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚665 G@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚77 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 5494538240 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17104896 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚25932805 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚223 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚322 _step‚678 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚19006574 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚6926156 SFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚73067520 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚225676515789895 IBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚60 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚227008540363776 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚225014950887424 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚262¦c¯x2Ä- _wandbruntime‚2211  _runtime‚2211…Ø¥‚rÅì /loss/FRt1/assignment_loss‚0.745552741999419 0loss/FRt2/assignment_loss‚0.6728186436560025 /loss/FRt1/detection_loss‚0.5578006820372691 epoch‚20 % _timestamp‚1.7866608633719542e+09 _step‚679 0loss/FRt2/detection_loss‚0.49404334530454475 %loss/total_loss‚2.470215412997235 trainer/global_step‚8249 _runtime‚2211.901831468r¦_dwÆñ /loss/FRt1/assignment_loss‚0.745552741999419 0loss/FRt2/assignment_loss‚0.6728186436560025 /loss/FRt1/detection_loss‚0.5578006820372691 epoch‚20 % _timestamp‚1.7866608633719542e+09 _step‚679 0loss/FRt2/detection_loss‚0.49404334530454475 %loss/total_loss‚2.470215412997235 trainer/global_step‚8249 _runtime‚2211.901831468§ Vd¡Ç‚(:– ‡ùÓ‘á gpu.0.gpu‚58.0 gpu.0.memory‚10, gpu.0.memoryAllocated‚7.0313262939453125) gpu.0.memoryAllocatedBytes‚ 6039863296 gpu.0.temp‚35.0 gpu.0.powerWatts‚194.032' gpu.0.enforcedPowerLimitWatts‚500.0) gpu.0.powerPercent‚38.806400000000004 gpu.0.smClock‚1410 gpu.0.memoryClock‚1593$ gpu.0.correctedMemoryErrors‚1955# gpu.0.uncorrectedMemoryErrors‚0ÝÍñ“žÈ‚(:“ ‡ùÓÄÉéâ disk.drbd2165.out‚0 disk.dm-0.in‚43395.58984375$ memory_percent‚66.98970794677734# proc.memory.rssMB‚ 2286.19921875 proc.cpu.threads‚17 network.sent‚ 1.0902995e+07 network.recv‚ 4.296087e+06* disk./.usagePercent‚32.691497270918724$ disk.nvme0c0n1.in‚43395.58984375 disk.sda1.in‚0 cpu‚27.113291945416275 disk.sda3.in‚ 21320.1171875 disk.sda.in‚ 21320.1171875 disk.sda1.out‚0 disk.sda2.in‚0" disk.nvme0n1.in‚43395.58984375& proc.memory.availableMB‚ 5408.40625* proc.memory.percent‚13.953852653503418( disk.nvme0c0n1.out‚55443.55029296875 disk.sda.out‚ 42.16015625 disk.sda2.out‚0& disk.nvme0n1.out‚55443.55029296875 disk.dm-1.in‚ 21320.1171875 disk.dm-1.out‚ 42.16015625 disk.drbd2165.in‚0# disk.dm-0.out‚55443.55029296875$ disk./.usageGB‚487.1269340515137 disk.sda3.out‚ 42.16015625¯b0 2É- _wandbruntime‚2224  _runtime‚222430 )ÇÊÁ & lr-AdamW/pg1‚0.0009750069944967825 & lr-AdamW/pg2‚0.0009750069944967825 trainer/global_step‚8299 % _timestamp‚1.7866608756083167e+09 _runtime‚2224.138327105 _step‚680‚eü/ÌËÆ & lr-AdamW/pg1‚0.0009750069944967825 & lr-AdamW/pg2‚0.0009750069944967825 trainer/global_step‚8299 % _timestamp‚1.7866608756083167e+09 _runtime‚2224.138327105 _step‚680¨3û›Ò2Ì- _wandbruntime‚2224  _runtime‚2224>¢Ï?ͶO NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚4287 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 IBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚60 M@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚59049984 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚19114619 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 2120025418240 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool¾N:X}%.allocated‚ 125829120 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚41945088 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚197292513722368 IDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚228309631559680 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 4654602240 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 2119972991488 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 2000725989583 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4289 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚262 F>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 4552982528 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 122905600 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚41945088 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚26081206 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚197292534366208 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 5368709120 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚199412486713856 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚262 IBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚82 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚52426752 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚262 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚27848670 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 5494538240 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 5368709120 _step‚681 PCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚59049984 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚27852957 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚228309690609664 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚27848670 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚58557198 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 $ _timestamp‚1.786660875609221e+09 C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚4289 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚226304745439232 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚223 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚20643840 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚61478912 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚226304728334336 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 2004945170432 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 845178880 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 5494538240 G>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4876 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚30704241 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚6966507 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚41348020 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 125829120 KDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚60 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚6966587 PCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚58387380 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚224970166534144 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 2004903225344 IDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚228309631559680 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 4654602240 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 2119972991488 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 2000725989583 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4289 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚262 F>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 4552982528 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 122905600 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚41945088 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚26081206 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚197292534366208 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 5368709120 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚199412486713856 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚262 IBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚82 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚52426752 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚262 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚27848670 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 5494538240 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 5368709120 _step‚681 PCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚59049984 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚27852957 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚228309690609664 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚27848670 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚58557198 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 $ _timestamp‚1.786660875609221e+09 C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚4289 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚226304745439232 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚223 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚20643840 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚61478912 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚226304728334336 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 2004945170432 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 845178880 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 5494538240 G>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4876 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚30704241 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚6966507 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚41348020 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 125829120 KDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚60 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚6966587 PCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚58387380 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚224970166534144 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 2004903225344 I8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚226997308889875 IBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚60 _runtime‚2224.302388289 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚228336244019200 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 K>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59053056 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 2005176242176 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚322 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚41948160 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 2120265555456 B9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚5264 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚44035072 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 2000997910263 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 4552982528 H@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚665 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚6967297 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 2005176242176 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚41948160 SFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚64678912 % _timestamp‚1.7866608757720695e+09 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚4295 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚30707792 I@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚4876 NADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚58387428 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚27851957 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚19116822 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚6967358 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 4517085184 G@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚60 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 5494538240 _step‚682 >5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5264 DDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521206 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 5494538240 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚27856250 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 2005134294016 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚226331067777024 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚58564044 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚4293 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚26084119 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚224996369367040 G@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚63 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 P6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚322 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 125829120 trainer/global_step‚8299 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚197315581739008 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 122306526 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚30707794 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 4552982528 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚209 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 125829120 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17104896 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚223 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚61478912 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 EDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654602240 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 D=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚60 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚20643840 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0{Z¯¦¦ ÑÆM >5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5264 DDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521206 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 5494538240 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚27856250 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 2005134294016 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚226331067777024 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚58564044 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚4293 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚26084119 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚224996369367040 G@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚63 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 P6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚322 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 125829120 trainer/global_step‚8299 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚197315581739008 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 122306526 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚30707794 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 4552982528 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚209 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 125829120 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17104896 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚223 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚61478912 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 EDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654602240 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 D=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚60 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚20643840 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚228336244019200 NADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚59053056 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚199435782615552 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚197315561095168 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚58564044 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 122905600 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚429 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 2120221520384 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 5494538240 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 5368709120 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 125829120 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 122905600 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚26084182 NGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚61 B8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚226997308889875 IBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚60 _runtime‚2224.302388289 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚228336244019200 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 K>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59053056 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 2005176242176 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚322 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚41948160 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 2120265555456 B9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚5264 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚44035072 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 2000997910263 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 4552982528 H@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚665 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚6967297 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 2005176242176 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚41948160 SFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚64678912 % _timestamp‚1.7866608757720695e+09 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚4295 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚30707792 I@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚4876 NADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚58387428 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚27851957 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚19116822 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚6967358 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 4517085184 G@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚60 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 5494538240 _step‚682ª¸¤ñ~2Ò- _wandbruntime‚2224  _runtime‚2224j‹ªlsÓí 0loss/FRt2/assignment_loss‚0.8892014592102676 /loss/FRt1/detection_loss‚0.5909430268468783 /loss/FRt2/detection_loss‚0.5443613576859035 &loss/total_loss‚3.0155487391947924 epoch‚20 trainer/global_step‚8299 _step‚683 % _timestamp‚1.7866608757730865e+09 _runtime‚2224.302919422 0loss/FRt1/assignment_loss‚0.9910428954517433°„ôtxÔò 0loss/FRt2/assignment_loss‚0.8892014592102676 /loss/FRt1/detection_loss‚0.5909430268468783 /loss/FRt2/detection_loss‚0.5443613576859035 &loss/total_loss‚3.0155487391947924 epoch‚20 trainer/global_step‚8299 _step‚683 % _timestamp‚1.7866608757730865e+09 _runtime‚2224.302919422 0loss/FRt1/assignment_loss‚0.9910428954517433« Hôû“Õ‚(:ˆ –ùÓþÝáà gpu.0.gpu‚0.0 gpu.0.memory‚0, gpu.0.memoryAllocated‚7.0313262939453125) gpu.0.memoryAllocatedBytes‚ 6039863296 gpu.0.temp‚34.0 gpu.0.powerWatts‚149.385' gpu.0.enforcedPowerLimitWatts‚500.0 gpu.0.powerPercent‚29.877 gpu.0.smClock‚1410 gpu.0.memoryClock‚1593$ gpu.0.correctedMemoryErrors‚1955# gpu.0.uncorrectedMemoryErrors‚0f“Q˜Ö‚(: –ùӤɿã disk.sda.in‚ 21320.1171875 disk.sda.out‚ 42.4140625 disk.sda1.out‚0 disk.sda2.out‚0" disk.nvme0n1.in‚43395.58984375& disk.nvme0n1.out‚55450.75830078125 network.recv‚ 4.324902e+06 disk.drbd2165.out‚0 disk.sda3.in‚ 21320.1171875 disk.sda2.in‚0 proc.cpu.threads‚17$ disk.nvme0c0n1.in‚43395.58984375( disk.nvme0c0n1.out‚55450.75830078125# memory_percent‚66.9931411743164& proc.memory.availableMB‚ 5407.84375# proc.memory.rssMB‚ 2286.19921875* proc.memory.percent‚13.953852653503418 cpu‚27.11351784562273 network.sent‚ 1.0974498e+07) disk./.usagePercent‚32.68849404653455$ disk./.usageGB‚487.0821838378906 disk.dm-1.in‚ 21320.1171875 disk.dm-1.out‚ 42.4140625 disk.drbd2165.in‚0 disk.sda3.out‚ 42.4140625 disk.sda1.in‚0 disk.dm-0.in‚43395.58984375# disk.dm-0.out‚55450.75830078125[˜[2×- _wandbruntime‚2236  _runtime‚2236MÈÃÇØÁ & lr-AdamW/pg1‚0.0009692660050379077 & lr-AdamW/pg2‚0.0009692660050379077 trainer/global_step‚8349 % _timestamp‚1.7866608878057592e+09 _runtime‚2236.335830487 _step‚684ØHÞÕÌÙÆ trainer/global_step‚8349 % _timestamp‚1.7866608878057592e+09 _runtime‚2236.335830487 _step‚684 & lr-AdamW/pg1‚0.0009692660050379077 & lr-AdamW/pg2‚0.0009692660050379077¬Ùl}2Ú- _wandbruntime‚2236  _runtime‚2236ÍRÒ½'Û·O ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 2012287800191 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4289 IDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 831258624 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 2132263543296 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 5494538240 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚262 M@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚59049984 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚41945088 D;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚5264 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚19224869 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 G>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4876 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚41348020 IBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚60 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 4552982528 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17104896 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 2016489991168 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 5368709120 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚262 F?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚60 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 2016531936256 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚227620845223936 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 5368709120 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚4289 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚41945088 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚322 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚226279473250304 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 122306526 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 E8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚58895661 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚30881889 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚229637394265088 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚198444903006208 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 5368709120 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚262 IDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚229637335215104 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚198444882362368 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚226279456210944 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 125829120 _runtime‚2236.336996929 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚228291744011135z PÂ'ܼO ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 2016531936256 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚200577145905664 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚61478912 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17104896 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚227620862328832 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚19224867 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 122905600 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚322 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 4654602240 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚28018059 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚228291802398515 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚4287 IBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚73 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 4654602240 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 4552982528 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚20643840 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 2012329148211 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚227620862328832 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚26232045 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚28013772 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚227620845223936 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚429 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 125829120 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 trainer/global_step‚8349 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚322 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚44038144 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚30881891 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 122905600 PCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚58387380 KDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚60 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚229637335215104 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚198444882362368 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚226279456210944 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 125829120 _runtime‚2236.336996929 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚228291744011135 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 2012287800191 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4289 IDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 831258624 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 2132263543296 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 5494538240 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚262 M@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚59049984 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚41945088 D;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚5264 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚19224869 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 G>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4876 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚41348020 IBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚60 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 4552982528 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17104896 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 2016489991168 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 5368709120 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚262 F?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚60 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 2016531936256 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚227620845223936 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 5368709120 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚4289 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚41945088 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚322 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚226279473250304 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 122306526 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 E8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚58895661 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚30881889 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚229637394265088 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚198444903006208 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 5368709120 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚262 IDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59053056 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 4654602240 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚229663888621568 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 2016763008000 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚200600504200704 BDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚4293 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚41948160 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚26235096 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 831258624 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521206 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚322 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚227647167561728 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚28021352 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 $ _timestamp‚1.786660887968677e+09 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚322 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 2016721059840 E8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚228318218764711 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 2012559720871 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚262 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚58906796 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚7008024 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 5494538240 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 NADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚59053056 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 SFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚64678912 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚229663947674624 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 5368709120 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 125829120 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚229663947674624 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚28021352 EDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚30885444 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 4517085184 >5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5264 IBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚60 G@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚60 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 5368709120 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚19227074 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚30885442 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 4552982528 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚198467929735168 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 5494538240 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 845178880 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚198467950379008 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚20643840 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 2132509786624 NADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚58387428 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 125829120 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚429 NGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚68 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚200600439521792 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚26235166 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚322 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 D=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚60 trainer/global_step‚8349 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚227647167561728 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 122905600 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 122905600 B9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚5264 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚19227072 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚228318160377283 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 2132553821696 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 122306526 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 _step‚686 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚4295 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚58906796 G@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚70 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚322 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654602240 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚41348068 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 5494538240 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚262 P8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚228318218764711 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 2012559720871 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚262 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚58906796 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚7008024 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 5494538240 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 NADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚59053056 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 SFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚64678912 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚229663947674624 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 5368709120 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 125829120 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚229663947674624 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚28021352 EDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚30885444 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 4517085184 >5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5264 IBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚60 G@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚60 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 5368709120 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚19227074 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚30885442 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 4552982528 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚198467929735168 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 5494538240 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 845178880 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚198467950379008 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚20643840 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 2132509786624 NADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚58387428 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 125829120 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚429 NGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚68 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚200600439521792 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚26235166 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚322 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 D=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚60 trainer/global_step‚8349 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚227647167561728 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 122905600 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 122905600 B9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚5264 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚19227072 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚228318160377283 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 2132553821696 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 122306526 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 _step‚686 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚4295 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚58906796 G@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚70 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚322 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654602240 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚41348068 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 5494538240 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚262 PDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59053056 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 4654602240 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚229663888621568 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 2016763008000 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚200600504200704 BDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚4293 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚41948160 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚26235096 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 831258624 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 DDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521206 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚322 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚227647167561728 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚28021352 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 $ _timestamp‚1.786660887968677e+09®f;€k2à- _wandbruntime‚2236  _runtime‚2236|ñùóráì 0loss/FRt2/assignment_loss‚0.6580865065033258 0loss/FRt2/detection_loss‚0.48020094367374644 % _timestamp‚1.7866608879696374e+09 _runtime‚ 2236.49944994 /loss/FRt1/detection_loss‚0.5158309479075849 %loss/total_loss‚2.425109754539327 epoch‚20 trainer/global_step‚8349 _step‚687 0loss/FRt1/assignment_loss‚0.7709913564546698w7,Ñwâñ %loss/total_loss‚2.425109754539327 epoch‚20 trainer/global_step‚8349 _step‚687 0loss/FRt1/assignment_loss‚0.7709913564546698 0loss/FRt2/assignment_loss‚0.6580865065033258 0loss/FRt2/detection_loss‚0.48020094367374644 % _timestamp‚1.7866608879696374e+09 _runtime‚ 2236.49944994 /loss/FRt1/detection_loss‚0.5158309479075849¯í:ÿù2ã- _wandbruntime‚2248  _runtime‚2248È'³ÇäÁ _step‚688 & lr-AdamW/pg1‚0.0009635109325332613 & lr-AdamW/pg2‚0.0009635109325332613 trainer/global_step‚8399 % _timestamp‚1.7866608999142547e+09 _runtime‚2248.444276529"‚ªÌåÆ & lr-AdamW/pg1‚0.0009635109325332613 & lr-AdamW/pg2‚0.0009635109325332613 trainer/global_step‚8399 % _timestamp‚1.7866608999142547e+09 _runtime‚2248.444276529 _step‚688°q£T&2æ- _wandbruntime‚2248  _runtime‚2248¦÷7¼'ç¶O C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚322 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚28183161 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚28183161 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 5368709120 M@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚59049984 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚229612595498543 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 4517085184 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 2144593394688 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 5494538240 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 IDeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4289 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚228936962113536 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚199597251002368 F>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 122905600 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 2028076756992 PCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚58387380 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 4654602240 IBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚79 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17104896 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚48232448 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17104896 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 4552982528 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚31059541 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚26383165 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚61478912 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚262 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 2028118702080 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚59242702 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚230965038870528 JBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚665 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚229612653885923 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚429 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚201741865040896 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 2023849610799 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚262 F?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚60 _step‚689 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚223 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚209 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚201741796164608 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 2023890958819 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 125829120 trainer/global_step‚8399 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚19335119 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 845178880 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚20643840 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚31059541 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚31059539 G>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4876 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 2028118702080 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚41945088 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚26383244 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 122306526 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 125829120 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚4287 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚19335117 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 831258624 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 4628521206 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚322 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 E8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚59238413 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 4552982528 PCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚59049984 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚31059539 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚28178874 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚28178874 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚230965097920512 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚7048048 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 UHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚68876288 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 @7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚5264 BDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚227588745887744‡œ¦~Üè»O SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 4552982528 PCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚59049984 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚31059539 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚28178874 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚28178874 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚230965097920512 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚7048048 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 UHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚68876288 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 @7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚5264 BDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚227588745887744 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚322 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocateÕœAåd‚28183161 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚28183161 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 5368709120 M@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚59049984 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚229612595498543 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 4517085184 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 2144593394688 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 5494538240 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 IDeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4289 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚228936962113536 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚199597251002368 F>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 122905600 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 2028076756992 PCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚58387380 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 4654602240 IBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚79 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17104896 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚48232448 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17104896 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 4552982528 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚31059541 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚26383165 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚61478912 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚262 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 2028118702080 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚59242702 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚230965038870528 JBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚665 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚229612653885923 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚429 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚201741865040896 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 2023849610799 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚262 F?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚60 _step‚689 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚223 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚209 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚201741796164608 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 2023890958819 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 125829120 trainer/global_step‚8399 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚19335119 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 845178880 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚20643840 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚31059541 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚31059539 G>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4876 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 2028118702080 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚41945088 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚26383244 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 122306526 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 125829120 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚4287 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚19335117 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 831258624 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 4628521206 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚322 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 E8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚59238413±”Ù2é- _wandbruntime‚2248  _runtime‚2248Ä\½oÇ&êÁM J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚31063092 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17104896 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚28186454 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚227614948720640 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 2024080183411 >5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5264 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚227614931681280 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 trainer/global_step‚8399 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚41348068 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 122905600 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚26386208 NADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚59053056 K>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59053056 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚228963301556224 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚59245253 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚31063092 G@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚60 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚59249548 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚20643840 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 D8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚4293 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚28182161 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 5494538240 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 D=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚60 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚28186454 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚4293 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚429 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 125829120 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚262 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚230991651330048 SFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚64678912 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚223 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚59249548 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚31063094 PDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521206 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 125829120 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚4295 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 2028349773824 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚230991592276992 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654602240 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚19337322 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 2144793666560 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 122306526 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚199620319019008 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 % _timestamp‚1.7866609000767252e+09 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚31063094 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚201765092041728 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 4654602240 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚209 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 125829120 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚322 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚7048886 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚199620298375168 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚44035072 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚262 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚262 E6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚322 _step‚690 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚41948160 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 4552982528 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚19337324 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚201765156720640 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚229639011864691 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 5368709120 IBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚60 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚228963284451328 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚59245253 NADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚58387428 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚322 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚228963301556224 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 122905600 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚26386278 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 5368709120 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚28182161 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 2144837701632 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚61478912 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 5494538240 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 5494538240 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚41948160 H@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚665 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚230991651330048 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 831258624 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 5368709120 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚322£fR€Ì&ëÆM C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚322 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚228963301556224 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 122905600 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚26386278 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 5368709120 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚28182161 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 2144837701632 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚61478912 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 5494538240 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 5494538240 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚41948160 H@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚665 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚230991651330048 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 831258624 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 5368709120 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚322 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚31063092 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17104896 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚28186454 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚227614948720640 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 2024080183411 >5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5264 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚227614931681280 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 trainer/global_step‚8399 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚41348068 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 122905600 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚26386208 NADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚59053056 K>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59053056 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚228963301556224 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚59245253 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚31063092 G@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚60 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚59249548 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚20643840 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 D8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚4293 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚28182161 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 5494538240 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 D=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚60 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚28186454 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚4293 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚429 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 125829120 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚262 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚230991651330048 SFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚64678912 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚223 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚59249548 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚31063094 PDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521206 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 125829120 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚4295 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 2028349773824 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚230991592276992 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654602240 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚19337322 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 2144793666560 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 122306526 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚199620319019008 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 % _timestamp‚1.7866609000767252e+09 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚31063094 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚201765092041728 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 4654602240 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚209 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 125829120 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚322 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚7048886 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚199620298375168 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚44035072 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚262 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚262 E6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚322 _step‚690 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚41948160 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 4552982528 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚19337324 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚201765156720640 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚229639011864691 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 5368709120 IBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚60 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚228963284451328 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚59245253 NADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚58387428²–rB2ì- _wandbruntime‚2248  _runtime‚22483³/¶síí 0loss/FRt1/assignment_loss‚0.6258693198518025 0loss/FRt2/assignment_loss‚0.6240926362457762 /loss/FRt2/detection_loss‚0.4670321995473737 trainer/global_step‚8399 _runtime‚2248.607521673 /loss/FRt1/detection_loss‚0.5781053596740761 &loss/total_loss‚2.2950995153190283 epoch‚20 % _timestamp‚1.7866609000777202e+09 _step‚691Ñœcâxîò /loss/FRt2/detection_loss‚0.4670321995473737 trainer/global_step‚8399 _runtime‚2248.607521673 /loss/FRt1/detection_loss‚0.5781053596740761 &loss/total_loss‚2.2950995153190283 epoch‚20 % _timestamp‚1.7866609000777202e+09 _step‚691 0loss/FRt1/assignment_loss‚0.6258693198518025 0loss/FRt2/assignment_loss‚0.6240926362457762³Â¥¤a”ï‚(:‰ ¥ùÓ¿Ñ­á gpu.0.gpu‚56.0 gpu.0.memory‚6, gpu.0.memoryAllocated‚7.0313262939453125) gpu.0.memoryAllocatedBytes‚ 6039863296 gpu.0.temp‚33.0 gpu.0.powerWatts‚119.305' gpu.0.enforcedPowerLimitWatts‚500.0 gpu.0.powerPercent‚23.861 gpu.0.smClock‚1410 gpu.0.memoryClock‚1593$ gpu.0.correctedMemoryErrors‚1955# gpu.0.uncorrectedMemoryErrors‚0ÅN¢Qð‚(:… ¥ùÓñÍã$ memory_percent‚66.99011325836182# proc.memory.rssMB‚ 2286.19921875* proc.memory.percent‚13.953852653503418 network.sent‚ 1.1046561e+07$ disk.nvme0c0n1.in‚43395.58984375 disk.sda1.in‚0 disk.dm-1.in‚ 21320.1171875) proc.memory.availableMB‚ 5408.33984375 cpu‚27.11594958725907 network.recv‚ 4.354278e+06 disk.sda3.out‚42.625( disk.nvme0c0n1.out‚55510.57861328125 disk.drbd2165.in‚0 disk.dm-0.in‚43395.58984375 proc.cpu.threads‚17% disk./.usageGB‚487.08266830444336 disk.sda3.in‚ 21320.1171875 disk.sda.in‚ 21320.1171875 disk.sda.out‚42.625 disk.sda1.out‚0 disk.sda2.in‚0 disk.sda2.out‚0" disk.nvme0n1.in‚43395.58984375( disk./.usagePercent‚32.6885265594914& disk.nvme0n1.out‚55510.57861328125 disk.dm-1.out‚42.625 disk.drbd2165.out‚0# disk.dm-0.out‚55510.57861328125ØNà‡2ñ- _wandbruntime‚2253  _runtime‚2253¡ˆ;ìÛòÕ /Purity/2FRt/event_purity‚0.4189504086971283 'particle/accuracy‚0.638974130153656 )loss/val_total_loss‚2.434423973487249 2Purity/*FRt/event_proportion‚0.669805645942688 3Purity/1FRt/event_proportion‚0.6297432780265808 +jet/accuracy_1_of_1‚0.31418177485466003 0Purity/*FRt/event_purity‚0.46862998604774475 2Purity/2FRt/event_proportion‚0.040062315762043 *jet/accuracy_1_of_2‚0.5838283896446228 'particle/f_score‚0.5436127185821533 trainer/global_step‚8399 _step‚692 -Purity/2FRt/FRt_purity‚0.5608131885528564 /particle/accuracy_2_of_2‚0.6586741209030151 *validation_accuracy‚0.2058325707912445 epoch‚20 3Purity/0FRt/event_proportion‚0.3301943838596344 .Purity/1FRt/FRt_purity‚0.47171396017074585 +particle/sensitivity‚0.6058855652809143 particle/accuracy_1_of_1‚1 *particle/specificity‚0.657181441783905 _runtime‚2253.604146378 % _timestamp‚1.7866609050741076e+09 Purity/0FRt/event_purity‚1 *jet/accuracy_2_of_2‚0.2058325707912445 /particle/accuracy_1_of_2‚0.9005352854728699 .Purity/*FRt/FRt_purity‚0.48184049129486084 0Purity/1FRt/event_purity‚0.47171396017074585 5validation_average_jet_accuracy‚0.318987637758255{´Î©àóÚ .Purity/*FRt/FRt_purity‚0.48184049129486084 0Purity/1FRt/event_purity‚0.47171396017074585 5validation_average_jet_accuracy‚0.318987637758255 /Purity/2FRt/event_purity‚0.4189504086971283 'particle/accuracy‚0.638974130153656 )loss/val_total_loss‚2.434423973487249 2Purity/*FRt/event_proportion‚0.669805645942688 3Purity/1FRt/event_proportion‚0.6297432780265808 +jet/accuracy_1_of_1‚0.31418177485466003 0Purity/*FRt/event_purity‚0.46862998604774475 2Purity/2FRt/event_proportion‚0.040062315762043 *jet/accuracy_1_of_2‚0.5838283896446228 'particle/f_score‚0.5436127185821533 trainer/global_step‚8399 _step‚692 -Purity/2FRt/FRt_purity‚0.5608131885528564 /particle/accuracy_2_of_2‚0.6586741209030151 *validation_accuracy‚0.2058325707912445 epoch‚20 3Purity/0FRt/event_proportion‚0.3301943838596344 .Purity/1FRt/FRt_purity‚0.47171396017074585 +particle/sensitivity‚0.6058855652809143 particle/accuracy_1_of_1‚1 *particle/specificity‚0.657181441783905 _runtime‚2253.604146378 % _timestamp‚1.7866609050741076e+09 Purity/0FRt/event_purity‚1 *jet/accuracy_2_of_2‚0.2058325707912445 /particle/accuracy_1_of_2‚0.9005352854728699´XžZ¡ô‚(:– ´ùÓ¬ç‚á gpu.0.gpu‚62.0 gpu.0.memory‚11, gpu.0.memoryAllocated‚7.0313262939453125) gpu.0.memoryAllocatedBytes‚ 6039863296 gpu.0.temp‚32.0 gpu.0.powerWatts‚119.166' gpu.0.enforcedPowerLimitWatts‚500.0) gpu.0.powerPercent‚23.833199999999998 gpu.0.smClock‚1410 gpu.0.memoryClock‚1593$ gpu.0.correctedMemoryErrors‚1955# gpu.0.uncorrectedMemoryErrors‚0%]Dœœõ‚(:‘ ´ùÓùËÕã disk.sda1.in‚0$ memory_percent‚66.99333190917969 network.recv‚ 4.386927e+06% disk./.usageGB‚487.03649139404297 disk.dm-0.in‚43417.25390625 disk.sda3.in‚21350.82421875 disk.sda2.out‚0# proc.memory.rssMB‚ 2286.19921875 cpu‚27.123328434076836% disk.nvme0n1.out‚55576.4462890625 disk.dm-1.out‚ 42.94140625 disk.drbd2165.in‚0 disk.sda3.out‚ 42.94140625 disk.sda1.out‚0 disk.sda2.in‚0% proc.memory.availableMB‚ 5407.8125* proc.memory.percent‚13.953852653503418" disk.nvme0n1.in‚43417.25390625 disk.dm-1.in‚21350.82421875" disk.dm-0.out‚55576.4462890625$ disk.nvme0c0n1.in‚43417.25390625 disk.sda.in‚21350.82421875 disk.sda.out‚ 42.94140625 proc.cpu.threads‚17 network.sent‚ 1.114221e+07) disk./.usagePercent‚32.68542758828942 disk.drbd2165.out‚0' disk.nvme0c0n1.out‚55576.4462890625“zY–2ö- _wandbruntime‚2266  _runtime‚2266‘¶OßÆ÷À & lr-AdamW/pg1‚0.0009577421466203962 & lr-AdamW/pg2‚0.0009577421466203962 trainer/global_step‚8449 % _timestamp‚1.7866609178091125e+09 _runtime‚ 2266.33919442 _step‚6934éØËøÅ & lr-AdamW/pg1‚0.0009577421466203962 & lr-AdamW/pg2‚0.0009577421466203962 trainer/global_step‚8449 % _timestamp‚1.7866609178091125e+09 _runtime‚ 2266.33919442 _step‚693µvÖÔ-2ù- _wandbruntime‚2266  _runtime‚2266.gŸqúµO @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚322 TGDeviceStatsMonitor.on_train_batch_sta3LE»J'rt/active_bytes.large_pool.current‚17104896 IDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚230644016218112 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚41945088 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚4287 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚223 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚262 E8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚59663963 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚31297797 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚28366168 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚201048889950208 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 2160966463488 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚28370455 JBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚665 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 2039518741533 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 125829120 KDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚60 _runtime‚2266.340406047 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚230644033323008 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚61478912 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 4517085184 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚232687747467264 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚201048910594048 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚322 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 122306526 F?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚60 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚232687806517248 IBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚78 UHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚75167744 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 125829120 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚232687747467264 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚322 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚4287 trainer/global_step‚8449 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚28366168 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 PIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚76 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚322 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 IBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚60 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚31297795 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚28370455 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚19485983 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚19485981 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 2039477393513 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚54523904 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚231318412449897 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 IDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4876 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17104896 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚41945088 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 4552982528 D;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚5264 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚429 BDeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4289 KBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚4876 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 2160911939584 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 5494538240 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚262 C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚4289 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 4552982528 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚7097977 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚41348020 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 5368709120 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚31297795 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚203209801889792 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 845178880 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 4628521206 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 4654602240 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚26583958 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 5494538240 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚229278952095744 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚230644033323008 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚230644016218112 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚20643840 PCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚58387380 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 5494538240 @7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚5264 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 4654602240 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚209 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚203209877057536 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 122905600 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 2043773194240 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚7098053 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 # _timestamp‚1.78666091781011e+09 M@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚59049984 PCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚59049984 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚26584036 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚229278935056384×ëÈÉÀ'ûºO 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚322 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 IBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚60 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚31297795 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚28370455 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚19485983 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚19485981 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 2039477393513 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚54523904 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚231318412449897 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 IDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4876 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17104896 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚41945088 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 4552982528 D;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚5264 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚429 BDeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4289 KBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚4876 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 2160911939584 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 5494538240 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚262 C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚4289 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 4552982528 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚7097977 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚41348020 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 5368709120 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚31297795 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚203209801889792 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 845178880 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 4628521206 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 4654602240 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚26583958 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 5494538240 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚229278952095744 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚230644033323008 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚230644016218112 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚20643840 PCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚58387380 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 5494538240 @7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚5264 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 4654602240 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚209 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚203209877057536 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 122905600 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 2043773194240 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚7098053 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 # _timestamp‚1.78666091781011e+09 M@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚59049984 PCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚59049984 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚26584036 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚229278935056384 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚322 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17104896 IDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚230644016218112 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚41945088 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚4287 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚223 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚262 E8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚59663963 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚31297797 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚28366168 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚201048889950208 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 2160966463488 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚28370455 JBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚665 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 2039518741533 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 125829120 KDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚60 _runtime‚2266.340406047 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚230644033323008 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚61478912 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 4517085184 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚232687747467264 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚201048910594048 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚322 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 122306526 F?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚60 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚232687806517248 IBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚78 UHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚75167744 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 125829120 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚232687747467264 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚322 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚4287 trainer/global_step‚8449 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚28366168 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 PIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚76¶êOò2ü- _wandbruntime‚2266  _runtime‚2266uìqÇ&ýÁM K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚26586976 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚28369455 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚230670338555904 % _timestamp‚1.7866609179763386e+09 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚59670803 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚41948160 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 2043962317824 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚201071957966848 trainer/global_step‚8449 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 2044004265984 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 2039707966125 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚322 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚31301350 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚7098790 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚52423680 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 IBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚60 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 4552982528 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 122905600 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚262 G@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚60 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚28373748 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654602240 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚203233168933888 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚232714359926784 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚232714300873728 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚232714359926784 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚4293 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 5368709120 _runtime‚2266.506602913 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚19488188 EDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521206 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 2044004265984 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚209 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚61478912 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 5368709120 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 2043962317824 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚7098863 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 831258624 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 122306526 I@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚4876 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚429 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚201071937323008 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 125829120 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚322 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚203233095866368 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 2039749314193 DDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59053056 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚231344887203473 NADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚58387428 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚230670338555904 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚28369455 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17104896 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚31301350 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 5368709120 >5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5264 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 4654602240 B9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚5264 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚231344828816045 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚230670355660800 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 4552982528 EDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 _step‚695 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚41948160 NGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚73 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚28373748 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚20643840 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚322 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚229305137889280 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚31301348 PDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚19488186 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚4295 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 125829120 H@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚665 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚322 D=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚60 SFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚73067520 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 5494538240 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚59670803 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 BDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚59675098 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚223 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 845178880 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 2161210967040 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 2161158543360 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚4293 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚31301348 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚229305120849920„.ÜoÚ þÆM NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚28373748 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654602240 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚203233168933888 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚232714359926784 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚232714300873728 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚232714359926784 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚4293 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 5368709120 _runtime‚2266.506602913 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚19488188 EDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521206 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 2044004265984 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚209 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚61478912 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 5368709120 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 2043962317824 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚7098863 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 831258624 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 122306526 I@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚4876 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚429 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚201071937323008 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 125829120 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚322 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚203233095866368 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 \JDeviceStatsMonit/Á³òor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 2039749314193 DDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59053056 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚231344887203473 NADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚58387428 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚230670338555904 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚28369455 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17104896 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚31301350 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 5368709120 >5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5264 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 4654602240 B9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚5264 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚231344828816045 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚230670355660800 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 4552982528 EDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 _step‚695 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚41948160 NGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚73 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚28373748 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚20643840 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚322 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚229305137889280 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚31301348 PDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚19488186 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚4295 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 125829120 H@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚665 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚322 D=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚60 SFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚73067520 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 5494538240 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚59670803 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 BDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚59675098 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚223 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 845178880 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 2161210967040 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 2161158543360 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚4293 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚31301348 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚229305120849920 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚26586976 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚28369455 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚230670338555904 % _timestamp‚1.7866609179763386e+09 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚59670803 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚41948160 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 2043962317824 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚201071957966848 trainer/global_step‚8449 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 2044004265984 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 2039707966125 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚322 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚31301350 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚7098790 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚52423680 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 IBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚60 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 4552982528 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 122905600 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚262 G@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚60·ž8æ¸2ÿ- _wandbruntime‚2266  _runtime‚2266+3ätp€ê &loss/total_loss‚2.2798614880862313 epoch‚21 _step‚696 0loss/FRt2/assignment_loss‚0.6450728662570854 trainer/global_step‚8449 # _timestamp‚1.78666091797742e+09 _runtime‚2266.507236574 0loss/FRt1/assignment_loss‚0.6328545948795682 /loss/FRt1/detection_loss‚0.5391741714080108 .loss/FRt2/detection_loss‚0.462759855541567´öø=uï 0loss/FRt1/assignment_loss‚0.6328545948795682 /loss/FRt1/detection_loss‚0.5391741714080108 .loss/FRt2/detection_loss‚0.462759855541567 &loss/total_loss‚2.2798614880862313 epoch‚21 _step‚696 0loss/FRt2/assignment_loss‚0.6450728662570854 trainer/global_step‚8449 # _timestamp‚1.78666091797742e+09 _runtime‚2266.507236574¸ŽíGþ2‚- _wandbruntime‚2278  _runtime‚2278;¹Â¦ÆƒÀ _step‚697 & lr-AdamW/pg1‚0.0009519600178176515 & lr-AdamW/pg2‚0.0009519600178176515 trainer/global_step‚8499 $ _timestamp‚1.786660929994943e+09 _runtime‚2278.524954485ÝRË„Å & lr-AdamW/pg1‚0.0009519600178176515 & lr-AdamW/pg2‚0.0009519600178176515 trainer/global_step‚8499 $ _timestamp‚1.786660929994943e+09 _runtime‚2278.524954485 _step‚697¹u,-ä2…- _wandbruntime‚2278  _runtime‚2278sR–í¼'†¶O \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚48232448 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚230588224733184 KDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚60 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚429 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 2055318014976 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚262 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚31475445 JBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚665 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 125829120 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚26734664 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚19596233 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚209 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 4517085184 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚26734739 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚28535557 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 4552982528 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 IBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚60 trainer/global_step‚8499 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 125829120 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚262 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚41348020 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚322 @7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚5264 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 UHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚68876288 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 2051080552141 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚322 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚262 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 4654602240 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 845178880 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 IDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 G>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4876 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚231960150212608 D;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚5264 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚204374448773632 BDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 5494538240 _runtime‚2278.529858638 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚31475447 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚4287 PIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚73 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 125829120 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚204374517649920 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 $ _timestamp‚1.786660929995847e+09 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚231960133107712 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚41945088 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 2055318014976 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚28531270 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚19596231 PCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚58387380 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 122905600 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚41945088 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚20643840 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚232639322324685 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 5368709120 M@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚59049984 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚231960133107712 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚7138433 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 831258624 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 4628521206 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚230588241772544 IDeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4289 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚7138506 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚202201258590208 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚322 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚4287 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚28531270 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚234015451122688 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚202201279234048 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 5494538240 E8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚60006715 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 2055359960064 IBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚75 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚60011004 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 2051039204121 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚4289bÓ YÁ'‡»O UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚234015451122688 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 122306526 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚223 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚322 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17104896 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 122905600 _step‚698 F>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 G>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4876 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚231960150212608 D;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚5264 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚204374448773632 BDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 5494538240 _runtime‚2278.529858638 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚31475447 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚4287 PIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚73 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 125829120 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚204374517649920 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 $ _timestamp‚1.786660929995847e+09 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚231960133107712 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚41945088 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 2055318014976 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚28531270 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚19596231 PCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚58387380 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 122905600 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚41945088 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚20643840 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚232639322324685 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 5368709120 M@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚59049984 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚231960133107712 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚7138433 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 831258624 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 4628521206 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚230588241772544 IDeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4289 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚7138506 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚202201258590208 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚322 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚4287 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚28531270 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚234015451122688 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚202201279234048 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 5494538240 E8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚60006715 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 2055359960064 IBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚75 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚60011004 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 2051039204121 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚4289 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚48232448 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚230588224733184 KDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚60 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚429 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 2055318014976 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚262 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚31475445 JBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚665 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 125829120 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚26734664 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚19596233 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚209 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 4517085184 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚26734739 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚28535557 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 4552982528 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 IBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚60 trainer/global_step‚8499 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 125829120 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚262 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚41348020 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚322 @7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚5264 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 UHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚68876288 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 2051080552141 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚322 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚262 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 4654602240 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 845178880 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 IDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654602240 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚4295 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚41948160 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 2055549083648 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚204397810378240 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚234042063582208 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 2055591031808 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚60017850 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚28534557 I@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚4876 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚202224326606848 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚202224305963008 G@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚60 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚322 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚231986455445504 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚31478998 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚429 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚41348068 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚262 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 845178880 trainer/global_step‚8499 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚231986472550400 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚46132224 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 5494538240 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17104896 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 2173437639168 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚322 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚322 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 5494538240 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚31479000 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚204397743602176 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 NADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚59053056 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 5368709120 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚60017850 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 2055591031808 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚26737737 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 2051269776733 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚262 D>DeviE=ÇceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 K>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59053056 EDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚230614410526720 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 125829120 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 4552982528 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚28538850 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 831258624 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚61478912 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚234042063582208 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚20643840 NADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚58387428 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 125829120 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚60013555 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚28534557 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚209 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 122905600 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521206 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 _step‚699 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚28538850 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 5368709120 H@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚665 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚7139236 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 B6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚322 D5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5264 P8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚262 D=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚60 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 4654602240 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 122905600 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚234042004529152 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚26737672 SFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚66776064 fO.Ë&ŠÅM EDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚230614410526720 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 125829120 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 4552982528 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚28538850 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 831258624 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚61478912 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚234042063582208 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚20643840 NADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚58387428 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 125829120 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚60013555 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚28534557 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚209 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 122905600 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521206 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 _step‚699 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚28538850 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 5368709120 H@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚665 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚7139236 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 B6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚322 D5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5264 P8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚262 D=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚60 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 4654602240 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 122905600 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚234042004529152 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚26737672 SFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚66776064 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654602240 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚4295 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚41948160 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 2055549083648 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚204397810378240 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚234042063582208 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 2055591031808 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚60017850 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚28534557 I@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚4876 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚202224326606848 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚202224305963008 G@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚60 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚322 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚231986455445504 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚31478998 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚429 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚41348068 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚262 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 845178880 trainer/global_step‚8499 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚231986472550400 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚46132224 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 5494538240 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17104896 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 2173437639168 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚322 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚322 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 5494538240 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚31479000 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚204397743602176 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 NADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚59053056 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 5368709120 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚60017850 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 2055591031808 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚26737737 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 2051269776733 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚262 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 K>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59053056»ƒ¯øÐ2‹- _wandbruntime‚2278  _runtime‚2278˜ÒˆÈsŒí /loss/FRt1/detection_loss‚0.5288383832539217 0loss/FRt2/detection_loss‚0.46575122222756843 epoch‚21 trainer/global_step‚8499 % _timestamp‚1.7866609301570687e+09 0loss/FRt2/assignment_loss‚0.6334764245513868 %loss/total_loss‚2.321722293274695 _runtime‚2278.687360911 _step‚700 0loss/FRt1/assignment_loss‚0.6936562632418181™‹7xò 0loss/FRt2/assignment_loss‚0.6334764245513868 %loss/total_loss‚2.321722293274695 _runtime‚2278.687360911 _step‚700 0loss/FRt1/assignment_loss‚0.6936562632418181 /loss/FRt1/detection_loss‚0.5288383832539217 0loss/FRt2/detection_loss‚0.46575122222756843 epoch‚21 trainer/global_step‚8499 % _timestamp‚1.7866609301570687e+09¼FŸc¡Ž‚(:– ÃùÓ²Š¢á gpu.0.gpu‚64.0 gpu.0.memory‚12, gpu.0.memoryAllocated‚7.0313262939453125) gpu.0.memoryAllocatedBytes‚ 6039863296 gpu.0.temp‚35.0 gpu.0.powerWatts‚201.564' gpu.0.enforcedPowerLimitWatts‚500.0) gpu.0.powerPercent‚40.312799999999996 gpu.0.smClock‚1410 gpu.0.memoryClock‚1593$ gpu.0.correctedMemoryErrors‚1955# gpu.0.uncorrectedMemoryErrors‚0›×F蛂(: ÃùÓöÞæã$ disk.nvme0c0n1.in‚43418.76953125 disk.sda.in‚22445.72265625 disk.sda1.out‚0( proc.memory.availableMB‚ 5408.1796875" disk.nvme0n1.in‚43418.76953125 disk.dm-1.in‚22445.72265625 disk.drbd2165.in‚0 disk.dm-0.in‚43418.76953125& disk.nvme0c0n1.out‚55579.861328125 disk.sda1.in‚0) disk./.usagePercent‚32.68543347646271 disk.dm-1.out‚ 43.14453125 disk.drbd2165.out‚0 disk.sda3.in‚22445.72265625 disk.sda.out‚ 43.14453125 disk.sda2.in‚0$ memory_percent‚66.99109077453613* proc.memory.percent‚13.953852653503418 network.sent‚ 1.118579e+07 network.recv‚ 4.406357e+06! disk.dm-0.out‚55579.861328125 disk.sda2.out‚0# proc.memory.rssMB‚ 2286.19921875 cpu‚27.123605483409953 proc.cpu.threads‚17$ disk./.usageGB‚487.0365791320801$ disk.nvme0n1.out‚55579.861328125 disk.sda3.out‚ 43.14453125*u}2- _wandbruntime‚2290  _runtime‚22905¬ ²Ç‘Á _step‚701 & lr-AdamW/pg1‚0.0009461649175003552 & lr-AdamW/pg2‚0.0009461649175003552 trainer/global_step‚8549 % _timestamp‚1.7866609422868931e+09 _runtime‚2290.818895451ÊB ±Ì’Æ & lr-AdamW/pg1‚0.0009461649175003552 & lr-AdamW/pg2‚0.0009461649175003552 trainer/global_step‚8549 % _timestamp‚1.7866609422868931e+09 _runtime‚2290.818895451 _step‚701½^aì72“- _wandbruntime‚2290  _runtime‚2290Eì€8½'”·O WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 2066904780800 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 122905600 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚26885627 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚223 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚262 F?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚60 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 2066904780800 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚31653095 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚209 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 5368709120 trainer/global_step‚8549 _runtime‚2290.819022601 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚233276249997312 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 2066946725888 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚262 % _timestamp‚1.7866609422875545e+09 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 PCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚58387380 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 2062601014729 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚28700659 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚4287 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚231897514409984 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 M@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚59049984 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4289 KBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚4876 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 4628521206 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 4654602240 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚7179146 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚322 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 5494538240 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚262 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17104896 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 2062642362749 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 5494538240 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 4552982528 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 4552982528 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 122905600 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚31653097 PCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚59049984 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 5494538240 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚322 BDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚19706483 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 845178880 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 @7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚5264 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚233276249997312 IDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4876 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚41945088 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 2185532741120 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚41348020 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 5368709120 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚322 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚322 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚235343154778112 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚233960115424713 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 125829120 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚41945088âMÍR•¼O M@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚59049984 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4289 KBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚4876 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 4628521206 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 4654602240 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚7179146 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚322 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 5494538240 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚262 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17104896 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 2062642362749 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 5494538240 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 4552982528 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 4552982528 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 122905600 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚31653097 PCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚59049984 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 5494538240 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚322 BDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚19706483 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 845178880 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 @7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚5264 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚233276249997312 IDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4876 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚41945088 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 2185532741120 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚41348020 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 5368709120 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚322 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚322 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚235343154778112 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚233960115424713 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 125829120 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚41945088 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 2066904780800 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 122905600 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚26885627 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚223 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚262 F?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚60 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 2066904780800 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚31653095 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚209 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 5368709120 trainer/global_step‚8549 _runtime‚2290.819022601 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚233276249997312 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 2066946725888 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚262 % _timestamp‚1.7866609422875545e+09 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 PCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚58387380 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 2062601014729 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚28700659 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚4287 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚231897514409984 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0¾ÂøGè2–- _wandbruntime‚2290  _runtime‚2290ÁªÃÇ&—ÁM UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17104896 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚19708688 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 2185734061568 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚26888681 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 2185775999488 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 NADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚58387428 SFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚62581760 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚322 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 4517085184 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 5368709120 _runtime‚2290.984031134 D6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚322 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚262 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 2062872935409 NADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚59053056 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚235369708184576 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚26888616 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 2067177797632 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 2067135849472 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚19708686 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚205562471246336 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚61478912 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 125829120 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚322 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚4295 EDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59053056 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚41937920 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 BDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521206 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚262 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚28703952 EDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 G@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚60 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 4654602240 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚41348068 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚60356307 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚60356307 I@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚4876 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚205562408664576 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚223 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 5368709120 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 2067177797632 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚28703952 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 5494538240 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 D=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚60 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 845178880 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17104896 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚233302589440000 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚41948160 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚60360602 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 5494538240 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 5494538240 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚60360602 G@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚65 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚429 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚20643840 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚231923700203520 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 2062831587341 >5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5264 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚233302589440000 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚233302572335104 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚7179930 _step‚703 PDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 % _timestamp‚1.7866609424536395e+09 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚31656648 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚4293 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 125829120 IBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚60 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 4552982528 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 122306526 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚235369767237632 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 122905600 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚235369767237632 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 4552982528 H@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚665 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚203376695246848 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚231923717242880 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚28699659 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚41948160 B9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚5264 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚31656650 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚7179993 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚203376674603008 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 5368709120 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654602240 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚209 øõlÌ&˜ÆM QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 845178880 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17104896 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚233302589440000 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚41948160 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚60360602 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 5494538240 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 5494538240 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚60360602 G@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚65 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚429 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚20643840 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚231923700203520 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 2062831587341 >5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5264 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚233302589440000 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚233302572335104 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚7179930 _step‚703 PDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 % _timestamp‚1.7866609424536395e+09 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚31656648 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚4293 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 125829120 IBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚60 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 4552982528 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 122306526 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚235369767237632 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 122905600 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚235369767237632 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 4552982528 H@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚665 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚203376695246848 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚231923717242880 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚28699659 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚41948160 B9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚5264 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚31656650 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚7179993 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚203376674603008 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 5368709120 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654602240 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚209 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17104896 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚19708688 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 2185734061568 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚26888681 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 2185775999488 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 NADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚58387428 SFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚62581760 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚322 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 4517085184 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 5368709120 _runtime‚2290.984031134 D6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚322 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚262 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 2062872935409 NADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚59053056 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚235369708184576 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚26888616 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 2067177797632 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 2067135849472 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚19708686 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚205562471246336 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚61478912 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 125829120 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚322 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚4295 EDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59053056 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚41937920 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 BDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521206 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚262 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚28703952 EDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 G@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚60 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 4654602240 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚41348068 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚60356307 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚60356307 I@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚4876 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚205562408664576 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚223 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 5368709120 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 2067177797632 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚28703952 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 5494538240 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 D=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚60¿'TÊS2™- _wandbruntime‚2290  _runtime‚2290EMršì 0loss/FRt2/detection_loss‚0.45075860883416413 trainer/global_step‚8549 _runtime‚2290.984571283 0loss/FRt1/assignment_loss‚0.7687451544909836 &loss/total_loss‚2.4836965943587255 epoch‚21 $ _timestamp‚1.786660942454756e+09 _step‚704 /loss/FRt2/assignment_loss‚0.697222489062729 /loss/FRt1/detection_loss‚0.5669703419708487K\¸9w›ñ /loss/FRt2/assignment_loss‚0.697222489062729 /loss/FRt1/detection_loss‚0.5669703419708487 0loss/FRt2/detection_loss‚0.45075860883416413 trainer/global_step‚8549 _runtime‚2290.984571283 0loss/FRt1/assignment_loss‚0.7687451544909836 &loss/total_loss‚2.4836965943587255 epoch‚21 $ _timestamp‚1.786660942454756e+09 _step‚704À‰® –œ‚(:‹ ÒùÓ¡Æ‘á gpu.0.gpu‚61.0 gpu.0.memory‚11, gpu.0.memoryAllocated‚7.0313262939453125) gpu.0.memoryAllocatedBytes‚ 6039863296 gpu.0.temp‚35.0 gpu.0.powerWatts‚218.198' gpu.0.enforcedPowerLimitWatts‚500.0 gpu.0.powerPercent‚43.6396 gpu.0.smClock‚1410 gpu.0.memoryClock‚1593$ gpu.0.correctedMemoryErrors‚1955# gpu.0.uncorrectedMemoryErrors‚0àù†s¢‚(:— Òùӡعä disk.sda3.out‚ 43.38671875( disk.nvme0c0n1.out‚55699.98291015625 disk.sda.out‚ 43.38671875 disk.drbd2165.in‚0* proc.memory.percent‚13.953852653503418 network.recv‚ 4.435049e+06 disk.sda.in‚22445.72265625 disk.sda1.in‚0" disk.nvme0n1.in‚43424.33984375 disk.dm-1.in‚22445.72265625 disk.dm-1.out‚ 43.38671875# memory_percent‚66.9956922531128) proc.memory.availableMB‚ 5407.42578125 disk.sda1.out‚0 disk.sda2.out‚0$ disk./.usageGB‚487.1469917297363 disk.sda3.in‚22445.72265625$ disk.nvme0c0n1.in‚43424.33984375 disk.drbd2165.out‚0 disk.dm-0.in‚43424.33984375# disk.dm-0.out‚55699.98291015625& disk.nvme0n1.out‚55699.98291015625# proc.memory.rssMB‚ 2286.19921875 cpu‚27.123223827449465 proc.cpu.threads‚17 disk.sda2.in‚0 network.sent‚ 1.1278117e+07) disk./.usagePercent‚32.69284335853376Ê… }2ž- _wandbruntime‚2304  _runtime‚2304Êÿ¹ÇŸÁ & lr-AdamW/pg1‚0.0009403572178769722 & lr-AdamW/pg2‚0.0009403572178769722 trainer/global_step‚8599 % _timestamp‚1.7866609555550127e+09 _runtime‚2304.085353525 _step‚705ÝN› Ì Æ & lr-AdamW/pg1‚0.0009403572178769722 & lr-AdamW/pg2‚0.0009403572178769722 trainer/global_step‚8599 % _timestamp‚1.7866609555550127e+09 _runtime‚2304.085353525 _step‚705ÁéK´2¡- _wandbruntime‚2304  _runtime‚2304úèèŒÓ ¢·O XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚19816733 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚7219637 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚206703854243328 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 5368709120 trainer/global_step‚8599 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 125829120 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚31830745 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚223 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚233206821126144 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17104896 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 4654602240 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚322 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚236670917483520 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 F>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚234592366886912 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚206703791658496 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚322 @7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚5264 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚31830747 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚31830745 IBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚65 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 2197795788288 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚235281025299501 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚262 UHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚62584832 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚322 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚233206804086784 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 4654602240 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 845178880 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚204506016514048 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚28861474 % _timestamp‚1.7866609555562704e+09 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚60696508 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚429 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 5494538240 _runtime‚2304.090078152 _step‚706 G>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4876 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17104896 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚41945088 60DeviceStatsMonitor.on_train_batcwêh_start/num_ooms‚0 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 5368709120 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚262 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚236670917483520 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 2078533491712 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 5494538240 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚234592383991808 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚41945088 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚209 C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚4289 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4289 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚20643840 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚204505995870208 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚234592383991808 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚19816731 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 BDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚236670858433536 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 F?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚60 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 2078491546624 JBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚665 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚61478912 PCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚58387380 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 4517085184 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚28865761 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚235280966912121 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚234592366886912{!þ¯Â'£¼O JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚4287 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚28861474 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚7219574 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 125829120 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 IBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚60 M@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚59049984 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 2078533491712 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚28865761 PIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚63 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚262 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚41940992 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 122306526 IDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚236670858433536 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 F?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚60 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 2078491546624 JBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚665 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚61478912 PCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚58387380 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 4517085184 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚28865761 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚235280966912121 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚234592366886912 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚19816733 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚7219637 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚206703854243328 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 5368709120 trainer/global_step‚8599 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 125829120 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚31830745 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚223 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚233206821126144 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17104896 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 4654602240 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚322 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚236670917483520 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 F>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚234592366886912 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚206703791658496 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚322 @7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚5264 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚31830747 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚31830745 IBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚65 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 2197795788288 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚235281025299501 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚262 UHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚62584832 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚322 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚233206804086784 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 4654602240 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 845178880 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚204506016514048 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚28861474 % _timestamp‚1.7866609555562704e+09 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚60696508 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚429 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 5494538240 _runtime‚2304.090078152 _step‚706 G>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4876 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17104896 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚41945088 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 5368709120 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚262 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚236670917483520 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 2078533491712 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 5494538240 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚234592383991808 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚41945088 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚209 C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚4289 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4289 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚20643840 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚204505995870208 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚234592383991808 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚19816731 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 BDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚27039311 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 2198036952576 E6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚322 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 _step‚707 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚41948160 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 4552982528 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 5368709120 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚4293 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚28869054 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚28869054 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚31834300 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚206727080195584 trainer/global_step‚8599 NADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚59053056 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17104896 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 122905600 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚233232989880320 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 5368709120 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 2074434746017 >5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5264 DDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59053056 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 831258624 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 125829120 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚322 IBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚60 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚234618689224704 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚19818936 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521206 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚262 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 122306526 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚60703354 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚4295 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚60699059 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚28864761 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚27039385 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 _runtime‚2304.251367279 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚236697411840000 I@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚4876 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 4517085184 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚429 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚209 % _timestamp‚1.7866609557207952e+09 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚31834300 B9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚5264 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚223 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚322 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚206727146971648 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚233233006919680 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 122905600 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚46132224 D=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚60 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 4552982528 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654602240 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 2078722615296 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚60703354 G@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚74 NGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚72 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚7220375 SFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚66776064 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚41948160 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 2198083084800 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚61478912 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚41348068 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 5494538240 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 5494538240 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 125829120 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17104896 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 845178880 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚262 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 4654602240 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚322 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚235307383278269 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 5368709120 E6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚322 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚235307383278269 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 5368709120 EDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚27039311 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 2198036952576 E6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚322 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 _step‚707 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚41948160 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 4552982528 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 5368709120 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚4293 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚28869054 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚28869054 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚31834300 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚206727080195584 trainer/global_step‚8599 NADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚59053056 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17104896 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 122905600 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚233232989880320 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 5368709120 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 2074434746017 >5DeviceStatsMonitor.on_train_batch_end/active.all.pm¢Ñ”eak‚5264 DDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59053056 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 831258624 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 125829120 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚322 IBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚60 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚234618689224704 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚19818936 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521206 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚262 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 122306526 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚60703354 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚4295 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚60699059 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚28864761 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚27039385 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 _runtime‚2304.251367279 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚236697411840000 I@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚4876 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 4517085184 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚429 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚209 % _timestamp‚1.7866609557207952e+09 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚31834300 B9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚5264 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚223 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚322 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚206727146971648 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚233233006919680 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 122905600 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚46132224 D=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚60 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 4552982528 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654602240 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 2078722615296 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚60703354 G@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚74Ã¥.!2§- _wandbruntime‚2304  _runtime‚2304äƒã r¨ì /loss/FRt1/assignment_loss‚0.668490677607157 /loss/FRt1/detection_loss‚0.5328442119458748 0loss/FRt2/detection_loss‚0.47361003521553163 %loss/total_loss‚2.276949870845619 % _timestamp‚1.7866609557220144e+09 _runtime‚2304.251876541 _step‚708 0loss/FRt2/assignment_loss‚0.6020049460770555 epoch‚21 trainer/global_step‚8599H)Òw©ñ /loss/FRt1/assignment_loss‚0.668490677607157 /loss/FRt1/detection_loss‚0.5328442119458748 0loss/FRt2/detection_loss‚0.47361003521553163 %loss/total_loss‚2.276949870845619 % _timestamp‚1.7866609557220144e+09 _runtime‚2304.251876541 _step‚708 0loss/FRt2/assignment_loss‚0.6020049460770555 epoch‚21 trainer/global_step‚8599Ä€2í“¡ª‚(:– áùÓ²ÁŸá gpu.0.gpu‚63.0 gpu.0.memory‚11, gpu.0.memoryAllocated‚7.0313262939453125) gpu.0.memoryAllocatedBytes‚ 6039863296 gpu.0.temp‚35.0 gpu.0.powerWatts‚227.888' gpu.0.enforcedPowerLimitWatts‚500.0) gpu.0.powerPercent‚45.577600000000004 gpu.0.smClock‚1410 gpu.0.memoryClock‚1593$ gpu.0.correctedMemoryErrors‚1955# gpu.0.uncorrectedMemoryErrors‚0[ŒÉ~›«‚(: áùÓÕ÷»ä disk.sda1.in‚0$ memory_percent‚66.99154376983643) proc.memory.availableMB‚ 5408.10546875 network.sent‚ 1.1347958e+07 network.recv‚ 4.462279e+06 disk.sda3.out‚ 43.609375 disk.sda.in‚22445.72265625 disk.sda2.out‚0" disk.nvme0n1.in‚43425.58203125 disk.dm-1.out‚ 43.609375# proc.memory.rssMB‚ 2286.19921875$ disk./.usageGB‚487.1558723449707 disk.sda3.in‚22445.72265625$ disk.nvme0c0n1.in‚43425.58203125 disk.drbd2165.out‚0 disk.dm-0.in‚43425.58203125# disk.dm-0.out‚55704.50244140625 proc.cpu.threads‚17 disk.sda1.out‚0 disk.sda2.in‚0& disk.nvme0n1.out‚55704.50244140625 disk.dm-1.in‚22445.72265625 disk.drbd2165.in‚0* proc.memory.percent‚13.953852653503418 cpu‚27.1231815001754) disk./.usagePercent‚32.69343934407347( disk.nvme0c0n1.out‚55704.50244140625 disk.sda.out‚ 43.609375Qh2¬- _wandbruntime‚2316  _runtime‚2316”ôè¤Æ­À trainer/global_step‚8649 % _timestamp‚1.7866609678404915e+09 _runtime‚ 2316.37052364 _step‚709 & lr-AdamW/pg1‚0.0009345372919651962 & lr-AdamW/pg2‚0.0009345372919651962”‰:ˮŠ& lr-AdamW/pg2‚0.0009345372919651962 trainer/global_step‚8649 % _timestamp‚1.7866609678404915e+09 _runtime‚ 2316.37052364 _step‚709 & lr-AdamW/pg1‚0.0009345372919651962Å`mÈ"2¯- _wandbruntime‚2316  _runtime‚2316‹ÏÐï¼'°¶O [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 2090120257536 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 2210129332736 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 4517085184 UHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚62584832 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚234516093763584 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 2090078312448 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 4654602240 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚235908500881408 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 4552982528 @7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚5264 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚19926983 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚223 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚205658385154048 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 5368709120 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚237998562088960 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚61039260 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚32008397 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚236601876786909 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 4628521206 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 5494538240 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚262 C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚4289 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚237998621138944 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚41940992 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚322 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚237998562088960 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 2210087391744 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 5494538240 F?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚60 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚19926981 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚235908483776512 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚4287 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 _runtime‚ 2316.37169481 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚32008397 JBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚665 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚207868451901952 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 M@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚59049984 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚41945088 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚27186956 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚205658364510208 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 IBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚68 PIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚66 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚322 IBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚60 E8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚61034971 G>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4876 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚235908500881408 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚20643840 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 125829120 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17104896 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 845178880 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 2085765983965 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚262 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚209 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚41348020 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 5368709120 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚322 trainer/global_step‚8649 % _timestamp‚1.7866609678414397e+09 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚235908483776512 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 KBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚4876 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4289 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 2085724635945 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 122905600 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚29030863 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚322 _step‚710 D;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚5264 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚7259973 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚4287 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚237998621138944 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17104896 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚27186888 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚236601818399529 IDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚29030863 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 4654602240 IDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚29030863 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 4654602240 IDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚237998562088960 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 2210087391744 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 5494538240 F?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚60 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚19926981 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚235908483776512 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚4287 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 _runtime‚ 2316.37169481 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚32008397 JBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚665 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚207868451901952 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 M@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚59049984 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚41945088 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚27186956 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚205658364510208 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 IBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚68 PIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚66 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚322 IBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚60 E8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚61034971 G>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4876 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚235908500881408 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚20643840 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 125829120 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17104896 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 845178880 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 2085765983965 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚262 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚209 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚41348020 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 5368709120 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚322 trainer/global_step‚8649 % _timestamp‚1.7866609678414397e+09 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚235908483776512 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 KBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚4876 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4289 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 2085724635945 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0ÆŸE;Ä2²- _wandbruntime‚2316  _runtime‚2316KXšv³ÁM WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚61478912 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 5494538240 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚32011948 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚235934823219200 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚262 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚29029863 K>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59053056 _step‚711 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚235934806114304 >5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5264 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚4295 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚61041811 H@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚665 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚29029863 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚262 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚32011948 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚20643840 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 5368709120 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 125829120 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 2090309381120 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚223 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚7260765 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 4517085184 NGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚59 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 D=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚60 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚61041811 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 845178880 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 125829120 trainer/global_step‚8649 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚4293 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚27189953 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚209 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 2085996556625 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚322 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚235934806114304 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚44035072 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 125829120 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 % _timestamp‚1.7866609680065777e+09 BDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚27189892 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚234542296596480 _runtime‚2316.536918957 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚322 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 122905600 XFDeviceStatsMonitor.on|Qÿ%¸_train_batch_end/requested_bytes.small_pool.freed‚ 2085955208557 IBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚60 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚32011950 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚238025174548480 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 831258624 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 5494538240 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 2090351329280 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚207891807641088 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚207891742962176 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 5368709120 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚61046106 G@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚61 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 122306526 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚41948160 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 I@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚4876 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚19929186 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚61046106 SFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚64678912 NADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚58387428 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚234542279557120 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 2090309381120 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 122905600 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚238025115495424 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚29034156 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 G@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚60 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 4654602240 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚32011950 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚322 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521206 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚322 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚205681411883008 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚235934823219200 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚7260706 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 2210375114240 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 5368709120 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚262 PDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17104896 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654602240.µÌ&´ÆM REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17104896 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654602240 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚61478912 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 5494538240 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚32011948 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚235934823219200 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚262 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚29029863 K>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59053056 _step‚711 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚235934806114304 >5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5264 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚4295 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚61041811 H@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚665 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚29029863 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚262 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚32011948 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚20643840 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 5368709120 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 125829120 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 2090309381120 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚223 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚7260765 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 4517085184 NGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚59 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 D=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚60 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚61041811 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 845178880 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 125829120 trainer/global_step‚8649 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚4293 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚27189953 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚209 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 2085996556625 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚322 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚235934806114304 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚44035072 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 125829120 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 % _timestamp‚1.7866609680065777e+09 BDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚27189892 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚234542296596480 _runtime‚2316.536918957 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚322 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 122905600 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 2085955208557 IBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚60 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚32011950 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚238025174548480 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 831258624 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 5494538240 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 2090351329280 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚207891807641088 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚207891742962176 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 5368709120 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚61046106 G@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚61 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 122306526 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚41948160 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 I@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚4876 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚19929186 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚61046106 SFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚64678912 NADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚58387428 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚234542279557120 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 2090309381120 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 122905600 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚238025115495424 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚29034156 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 G@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚60 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 4654602240 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚32011950 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚322 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521206 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚322 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚205681411883008 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚235934823219200 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚7260706 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 2210375114240 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 5368709120 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚262 PDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0Çd„QÞ2µ- _wandbruntime‚2316  _runtime‚2316vÅšÅs¶í _step‚712 &loss/total_loss‚2.3414570792954064 epoch‚21 0loss/FRt1/assignment_loss‚0.6897572628145133 0loss/FRt2/assignment_loss‚0.6085108306906568 /loss/FRt1/detection_loss‚0.5596757247091464 /loss/FRt2/detection_loss‚0.4835132610810898 trainer/global_step‚8649 % _timestamp‚1.7866609680076847e+09 _runtime‚2316.537558895›ê’x·ò 0loss/FRt1/assignment_loss‚0.6897572628145133 0loss/FRt2/assignment_loss‚0.6085108306906568 /loss/FRt1/detection_loss‚0.5596757247091464 /loss/FRt2/detection_loss‚0.4835132610810898 trainer/global_step‚8649 % _timestamp‚1.7866609680076847e+09 _runtime‚2316.537558895 _step‚712 &loss/total_loss‚2.3414570792954064 epoch‚21È|?Û” ¸‚(:• ðùÓÀé¯á gpu.0.gpu‚38.0 gpu.0.memory‚9, gpu.0.memoryAllocated‚7.0313262939453125) gpu.0.memoryAllocatedBytes‚ 6039863296 gpu.0.temp‚33.0 gpu.0.powerWatts‚119.166' gpu.0.enforcedPowerLimitWatts‚500.0) gpu.0.powerPercent‚23.833199999999998 gpu.0.smClock‚1410 gpu.0.memoryClock‚1593$ gpu.0.correctedMemoryErrors‚1955# gpu.0.uncorrectedMemoryErrors‚0nçá×–¹‚(:‹ ðùÓÑýä disk.dm-0.out‚ 55713.2890625# proc.memory.rssMB‚ 2286.19921875* proc.memory.percent‚13.953852653503418 cpu‚27.123044994286538 network.sent‚ 1.142145e+07) disk./.usagePercent‚32.69346545684196 disk.sda.in‚22445.72265625 disk.sda2.out‚0 disk.dm-1.out‚ 44.01953125 disk.drbd2165.in‚0 disk.dm-0.in‚43438.23828125 disk.sda3.in‚22445.72265625 network.recv‚ 4.493033e+06$ disk./.usageGB‚487.1562614440918$ disk.nvme0c0n1.out‚ 55713.2890625 disk.sda1.in‚0" disk.nvme0n1.in‚43438.23828125" disk.nvme0n1.out‚ 55713.2890625 disk.sda3.out‚ 44.01953125) proc.memory.availableMB‚ 5407.81640625$ disk.nvme0c0n1.in‚43438.23828125 disk.sda.out‚ 44.01953125 disk.sda1.out‚0 disk.sda2.in‚0 disk.dm-1.in‚22445.72265625$ memory_percent‚66.99330806732178 proc.cpu.threads‚17 disk.drbd2165.out‚0ÁŸ¯Ë2º- _wandbruntime‚2328  _runtime‚2328?ù–\Ç»Á trainer/global_step‚8699 % _timestamp‚1.7866609801966062e+09 _runtime‚2328.726886387 _step‚713 & lr-AdamW/pg1‚0.0009287055135679935 & lr-AdamW/pg2‚0.0009287055135679935'äA.Ì¼Æ % _timestamp‚1.7866609801966062e+09 _runtime‚2328.726886387 _step‚713 & lr-AdamW/pg1‚0.0009287055135679935 & lr-AdamW/pg2‚0.0009287055135679935 trainer/global_step‚8699É:^ÅÑ2½- _wandbruntime‚2328  _runtime‚2328ãž¼'¾¶O SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 4552982528 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚239326324794368 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚32186045 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 125829120 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚262 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚29195965 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚237922728274317 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 2101665078272 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚235825383440384 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚322 IBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚60 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚20643840 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 4517085184 M@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚59049984 E8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚61377723 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 F?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚60 $ _timestamp‚1.786660980197771e+09 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 5368709120 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 125829120 _runtime‚2328.731879779 D;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚5264 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4289 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚29195965 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚4287 G>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4876 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 2222387186688 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚20037233 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚223 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚7300959 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚206810753794048 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚239326324794368 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚262 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚29191678 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚7300884 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 122306526 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 5368709120 F>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 2101707023360 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚235825400479744 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚322 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚27338115 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 2222343148544 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚61478912 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 IBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚77 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 4654602240 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚237224600666112 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚29191678 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 831258624 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚322 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚41348020 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚237224617771008 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 122905600 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚237224617771008 PIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚75 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 2097286446553 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚262 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 4654602240 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚209033076298752 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 845178880 IDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚239326265744384 IDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚239326265744384 IDeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4289 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚29195965 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚4287 G>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4876 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 2222387186688 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚20037233 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚223 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚7300959 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚206810753794048 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚239326324794368 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚262 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚29191678 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚7300884 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 122306526 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 5368709120 F>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 2101707023360 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚235825400479744 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚322 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚27338115 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 2222343148544 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚61478912ÊI³TÒ2À- _wandbruntime‚2328  _runtime‚2328ù[ ÉÇ&ÁÁM A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚322 D=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚60 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚29199258 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 831258624 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521206 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 2101938095104 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 2101896146944 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚223 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 _runtime‚2328.888595122 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 4654602240 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 122905600 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚41948160 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚237949086253085 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 5494538240 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 125829120 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚262 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚61388858 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚206833801166848 IBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚60 % _timestamp‚1.7866609803583217e+09 K>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59053056 SFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚68873216 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚237949144640513 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 2097558367233 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚7301782 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚7301713 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚27341220 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚209056437379072 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 5368709120 >5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5264 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚237250940108800 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 4517085184 trainer/global_step‚8699 EDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654602240 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚61384563 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚4293 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚48229376 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 B6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚322 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 DDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚239352878203904 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚322 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚237250940108800 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚61388858 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚27341149 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚209056368505856 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚239352878203904 NGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚69 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚206833780523008 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚235851586273280 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 122306526 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚262 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚237250923003904 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 2097517019165 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚262 _step‚715 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚32189600 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 2222636212224 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 122905600 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 5494538240 E6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚322 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚41348068 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚32189600 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚20039438 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚29194965 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚237250923003904 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 5368709120âXíÐÌ&ÂÆM O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 5494538240 E6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚322 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚41348068 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚32189600 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚20039438 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚29194965 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚237250923003904 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 5368709120 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚322 D=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚60 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚29199258 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 831258624 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521206 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 2101938095104 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 2101896146944 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚223 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 _runtime‚2328.888595122 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 4654602240 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 122905600 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚41948160 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚237949086253085 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 5494538240 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 125829120 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚262 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚61388858 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚206833801166848 IBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚60 % _timestamp‚1.7866609803583217e+09 K>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59053056 SFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚68873216 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚237949144640513 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 2097558367233 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚7301782 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚7301713 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚27341220 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚209056437379072 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 5368709120 >5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5264 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚237250940108800 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 4517085184 trainer/global_step‚8699 EDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654602240 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚61384563 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚4293 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚48229376 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 B6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚322 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 DDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚239352878203904 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚322 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚237250940108800 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚61388858 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚27341149 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚209056368505856 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚239352878203904 NGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚69 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚206833780523008 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚235851586273280 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 122306526 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚262 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚237250923003904 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 2097517019165 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚262 _step‚715 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚32189600 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 2222636212224 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 122905600Ë=Ä͘2Ã- _wandbruntime‚2328  _runtime‚2328ˆ±fssÄí /loss/FRt1/detection_loss‚0.5854010873918791 epoch‚21 0loss/FRt2/assignment_loss‚0.6155774739883755 /loss/FRt2/detection_loss‚0.4392823667674311 &loss/total_loss‚2.5152526461540017 trainer/global_step‚8699 % _timestamp‚1.7866609803593173e+09 _runtime‚2328.889118942 _step‚716 0loss/FRt1/assignment_loss‚0.8749917180063161ù6xÅò /loss/FRt1/detection_loss‚0.5854010873918791 epoch‚21 0loss/FRt2/assignment_loss‚0.6155774739883755 /loss/FRt2/detection_loss‚0.4392823667674311 &loss/total_loss‚2.5152526461540017 trainer/global_step‚8699 % _timestamp‚1.7866609803593173e+09 _runtime‚2328.889118942 _step‚716 0loss/FRt1/assignment_loss‚0.8749917180063161Ìr)î¡Æ‚(:– ÿùÓéßíà gpu.0.gpu‚63.0 gpu.0.memory‚12, gpu.0.memoryAllocated‚7.0313262939453125) gpu.0.memoryAllocatedBytes‚ 6039863296 gpu.0.temp‚34.0 gpu.0.powerWatts‚180.002' gpu.0.enforcedPowerLimitWatts‚500.0) gpu.0.powerPercent‚36.000400000000006 gpu.0.smClock‚1410 gpu.0.memoryClock‚1593$ gpu.0.correctedMemoryErrors‚1955# gpu.0.uncorrectedMemoryErrors‚0JÁ;à’Ç‚(:‡ ÿùÓÏÁµã# proc.memory.rssMB‚ 2286.19921875$ disk./.usageGB‚487.1562385559082" disk.nvme0n1.in‚43438.23828125% disk.nvme0n1.out‚55716.7392578125 disk.dm-1.in‚22445.72265625" disk.dm-0.out‚55716.7392578125$ disk.nvme0c0n1.in‚43438.23828125* proc.memory.percent‚13.953852653503418 cpu‚27.123544957481293 network.sent‚ 1.1478412e+07 disk.sda.in‚22445.72265625 disk.drbd2165.out‚0 disk.sda3.in‚22445.72265625 proc.cpu.threads‚17 disk.sda.out‚44.4375 disk.sda1.in‚0 disk.sda1.out‚0 disk.sda2.in‚0 disk.drbd2165.in‚0 disk.dm-0.in‚43438.23828125 disk.sda3.out‚44.4375$ memory_percent‚66.99204444885254 network.recv‚ 4.50604e+06) disk./.usagePercent‚32.69346392079676 disk.sda2.out‚0 disk.dm-1.out‚44.4375' disk.nvme0c0n1.out‚55716.7392578125( proc.memory.availableMB‚ 5408.0234375—"Ô@2È- _wandbruntime‚2340  _runtime‚2340Zu7(ÇÉÁ & lr-AdamW/pg2‚0.0009228622572495928 trainer/global_step‚8749 % _timestamp‚1.7866609923808832e+09 _runtime‚2340.912927815 _step‚717 & lr-AdamW/pg1‚0.0009228622572495928„î½6ÌÊÆ % _timestamp‚1.7866609923808832e+09 _runtime‚2340.912927815 _step‚717 & lr-AdamW/pg1‚0.0009228622572495928 & lr-AdamW/pg2‚0.0009228622572495928 trainer/global_step‚8749ÍãUM 2Ë- _wandbruntime‚2340  _runtime‚2340f}vì̵O O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 4628521206 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 _runtime‚ 2340.91303103 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 4552982528 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 2108889605181 _step‚718 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚32363697 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚240654028449792 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚32363697 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚207963122434048 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17104896 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚7341323 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚46135296 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚61478912 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 125829120 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚29356780 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 2113251844096 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚61724764 UHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚66779136 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 831258624 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 PCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚58387380 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚240653969399808 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚238540734660608 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚29361067 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 122306526 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚32363695 G>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4876 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 2113251844096 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚240654028449792 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚238540717555712 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 2113293789184 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17104896 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚27488806 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 $ _timestamp‚1.786660992381543e+09 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 4654602240 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚32363695 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚209 IDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚240653969399808 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚4287 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚27488737 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 G?DeviceStatsMonitor.on_train_bap‚ŽÏtch_start/inactive_split.all.peak‚429 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚20643840 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚237134673117184 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚29361067 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚322 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚322 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚239243579761725 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 125829120 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 4654602240 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚41945088 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 5368709120 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 KDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚60 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 122905600 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚7341256 trainer/global_step‚8749 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚210197719954944 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚237134690156544 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 M@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚59049984 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 2234664300032 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚207963101790208 D;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚5264 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 KBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚4876 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚262 @7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚5264 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 2234618164736 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 E8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚61720475 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4289 IDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚238540717555712 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚210197786734080 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 5368709120 IBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚60 F?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚60 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚238540734660608 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚20147481 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚239243521374345 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 5494538240 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚41945088 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚223 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 2108848257161 C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚4289 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0¢Ì³©À'ͺO L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 4654602240 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚41945088 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 5368709120 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 KDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚60 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 122905600 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚7341256 trainer/global_step‚8749 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚210197719954944 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚237134690156544 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 M@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚59049984 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 2234664300032 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚207963101790208 D;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚5264 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 KBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚4876 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚262 @7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚5264 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 2234618164736 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 E8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚61720475 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4289 IDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚238540717555712 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚210197786734080 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 5368709120 IBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚60 F?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚60 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚238540734660608 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚20147481 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚239243521374345 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 5494538240 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚41945088 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚223 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 2108848257161 C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚4289 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 4628521206 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 _runtime‚ 2340.91303103 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 4552982528 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 2108889605181 _step‚718 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚32363697 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚240654028449792 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚32363697 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚207963122434048 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17104896 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚7341323 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚46135296 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚61478912 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 125829120 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚29356780 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 2113251844096 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚61724764 UHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚66779136 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 831258624 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 PCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚58387380 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚240653969399808 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚238540734660608 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚29361067 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 122306526 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚32363695 G>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4876 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 2113251844096 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚240654028449792 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚238540717555712 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 2113293789184 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17104896 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚27488806 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 $ _timestamp‚1.786660992381543e+09 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 4654602240 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚32363695 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚209 IDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚240653969399808 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚4287 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚27488737 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚429 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚20643840 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚237134673117184 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚29361067 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚322 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚322 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚239243579761725 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 125829120ίdžÇ2Î- _wandbruntime‚2341  _runtime‚2341L!ÈÇ&ÏÁM S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚239269937740493 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚52423680 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 2109078829773 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 5368709120 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚322 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚210221079986688 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 5494538240 D=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚60 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚237160858910720 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 125829120 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚4295 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654602240 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚20149686 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 845178880 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 125829120 trainer/global_step‚8749 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚4293 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 2113524860928 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 122905600 H@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚665 SFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚73067520 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 4654602240 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚209 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚20149688 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚29360067 EDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521206 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 4517085184 _runtime‚2341.073757472 NGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚75 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚262 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17104896 G@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚77 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚61731610 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚4293 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚238567039893504 B9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚5264 G@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚60 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚29364360 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚7342055 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚210221006919168 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 4552982528 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 2113482912768 I@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚4876 K>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59053056 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚240680522806272 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 B6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚322 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚322 >5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5264 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 4552982528 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚27491741 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚32367248 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚41948160 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚61727315 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚207986149163008 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 831258624 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 2113524860928 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚223 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 2109120177841 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 5368709120 IBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚60 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚61731610 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚27491818 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚429 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚41348068 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚238567056998400 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚238567056998400 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚7342130 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚239269996127921 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚262 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚262 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 2113482912768 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 122905600 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 125829120 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚322 % _timestamp‚1.7866609925434391e+09 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚238567039893504 NADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚59053056 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚32367250 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚207986169806848 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚20643840 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 D6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚322 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚210221079986688 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 5494538240 D=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚60 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚237160858910720 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 125829120 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚4295 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654602240 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚20149686 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 845178880 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 125829120 trainer/global_step‚8749 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚4293 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 2113524860928 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 122905600 H@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚665 SFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚73067520 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 4654602240 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚209 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚20149688 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚29360067 EDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521206 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 4517085184 _runtime‚2341.073757472 NGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚75 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚262 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17104896 G@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚77 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚61731610 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚4293 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚238567039893504 B9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚5264 G@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚60 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚29364360 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚7342055 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚210221006919168 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 4552982528 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 2113482912768 I@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚4876 K>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59053056 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚240680522806272 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 B6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚322 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚322 >5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5264 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 4552982528 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚27491741 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚32367248 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚41948160 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚61727315 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚207986149163008 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 831258624 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 2113524860928 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚223 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 2109120177841 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 5368709120 IBDeviceStats;BŽPwMonitor.on_train_batch_end/segment.small_pool.allocated‚60 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚61731610 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚27491818 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚429 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚41348068 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚238567056998400 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚238567056998400 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚7342130 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚239269996127921 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚262 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚262 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 2113482912768 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 122905600 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 125829120 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚322 % _timestamp‚1.7866609925434391e+09 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚238567039893504 NADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚59053056 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚32367250 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0Ï7·´ä2Ñ- _wandbruntime‚2341  _runtime‚2341ÀÀ$!tÒî 0loss/FRt2/assignment_loss‚0.6967393973576685 /loss/FRt1/detection_loss‚0.5159833062690684 0loss/FRt2/detection_loss‚0.49500254651119335 epoch‚21 trainer/global_step‚8749 % _timestamp‚1.7866609925444744e+09 0loss/FRt1/assignment_loss‚0.7282558785720729 &loss/total_loss‚2.4359811287100035 _runtime‚2341.074303873 _step‚7207FtÃyÓó 0loss/FRt1/assignment_loss‚0.7282558785720729 &loss/total_loss‚2.4359811287100035 _runtime‚2341.074303873 _step‚720 0loss/FRt2/assignment_loss‚0.6967393973576685 /loss/FRt1/detection_loss‚0.5159833062690684 0loss/FRt2/detection_loss‚0.49500254651119335 epoch‚21 trainer/global_step‚8749 % _timestamp‚1.7866609925444744e+09Ѐ~‹2Ô- _wandbruntime‚2352  _runtime‚2352ˆdPXÇÕÁ % _timestamp‚1.7866610044670494e+09 _runtime‚2352.997089645 _step‚721 & lr-AdamW/pg1‚0.0009170078983114283 & lr-AdamW/pg2‚0.0009170078983114283 trainer/global_step‚8799È|® ÌÖÆ % _timestamp‚1.7866610044670494e+09 _runtime‚2352.997089645 _step‚721 & lr-AdamW/pg1‚0.0009170078983114283 & lr-AdamW/pg2‚0.0009170078983114283 trainer/global_step‚8799Ñ ü‡Ó2×- _wandbruntime‚2353  _runtime‚2353PÈð¼'ضO IDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚241981673055232 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚239856851550208 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚61478912 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚322 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 5494538240 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚29526169 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚209115491074048 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17104896 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 4654602240 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚7382230 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 @7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚5264 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚32541347 G>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4876 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 5368709120 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚41945088 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚41348020 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚239856834445312 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚20257733 F>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚211362444505600 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚238443979833344 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚238443962793984 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚29521882 JBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚665 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚20643840 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 4628521206 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 125829120 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚262 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 4517085184 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 5494538240 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚262 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 122905600 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 2124838609920 KBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚4876 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚52426752 BDeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4289 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚29521882 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚27639885 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚223§5ãÁ'Ù»O YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚239856834445312 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚20257733 F>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚211362444505600 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚238443979833344 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚238443962793984 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚29521882 JBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚665 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚20643840 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 4628521206 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 125829120 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚262 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 4517085184 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 5494538240 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚262 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 122905600 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 2124838609920 KBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚4876 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚52426752 BDeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4289 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚29521882 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚27639885 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚223 IDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚241981673055232 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚239856851550208 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚61478912 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚322 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 5494538240 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚29526169 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚209115491074048 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17104896 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 4654602240 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚7382230 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 @7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚5264 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚32541347 G>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4876 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 5368709120 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚41945088 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚41348020Ò"Ë­2Ú- _wandbruntime‚2353  _runtime‚2353Kmp#ÛÀM MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 831258624 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚262 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 4552982528 G@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚61 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚20643840 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 125829120 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚322 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚239883173888000 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚29525169 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 845178880 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚62074362 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚4295 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17104896 B9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚5264 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚29525169 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚238470165626880 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 2120681988449 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚322 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚209 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 2247151442432 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚61478912 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚242008285514752 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚242008285514752 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚239883156783104 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 2125111626752 H@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚665 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚209138517803008 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚32544900 K>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59053056 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚240590847615329 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 5368709120 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 125829120 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚29529462 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚27642891 NGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚59 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 BDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚62074362 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚20259936 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚322 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 122905600 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 5368709120 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 trainer/global_step‚8799 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 2125069678592 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 122306526 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚429 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚239883156783104 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 EDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521206 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚242008226461696 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚44035072 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚7382955 SFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚64678912 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 IBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚60 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654602240 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚322 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 D5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5264 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚32544898 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚7383014 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 2247195477504 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 J=DeviceStatsMonitor.on_Ãug¼Vtrain_batch_end/active.large_pool.freed‚32544898 PDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59053056 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚240590847615329 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 5368709120 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 125829120 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚29529462 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚27642891 NGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚59 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 BDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚62074362 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚20259936 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚322 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 122905600 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 5368709120 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 trainer/global_step‚8799 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 2125069678592 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 122306526 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚429 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚239883156783104 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 EDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521206 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚242008226461696 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚44035072 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚7382955 SFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚64678912 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 IBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚60 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654602240 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚322 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 D5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5264 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚32544898 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚7383014 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 2247195477504 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚32544898 PDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚242270248200482 F?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚60 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚223 E8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚62488772 @7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚5264 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚29709171 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚29713458 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚32779601 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚243704381649408 PCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚59049984 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 4552982528 IBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚75 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚7431785 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚240134151962624 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚241563905654784 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 4517085184 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 5368709120 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚29713458 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚210567130021888 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚242270189813102 G>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4876 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚241563888549888 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 4654602240 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚62493061 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 2263326104064 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚50329600 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 122905600 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚243704440699392 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚4287 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚32779603 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚241563905654784 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚210567109378048 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚322 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚322 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚4287 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚243704440699392 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚61478912 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 4628521206 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚262 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 125829120 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 2140535044608 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚32779601 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 5494538240 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚262 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚27840307 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚209 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚41945088 KBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚4876 KDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚60 JBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚665 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 845178880 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚243704381649408 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚20643840 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚20408595 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚322 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17104896 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 2140493099520 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4289 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 5494538240 trainer/global_step‚8849 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚29709171 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 2140493099520 UHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚70973440 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚240134169001984 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚7431712 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚262 _step‚727 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0pO'zHì¼O C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚4289 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚241563888549888 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚41945088 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚41348020 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚322 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 5368709120 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 2136079198498 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 122306526 _runtime‚2370.523970242 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 122905600 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚27840382 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚212830385152512 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 D;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚5264 BDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚242270248200482 F?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚60 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚223 E8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚62488772 @7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚5264 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚29709171 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚29713458 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚32779601 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚243704381649408 PCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚59049984 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 4552982528 IBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚75 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚7431785 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚240134151962624 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚241563905654784 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 4517085184 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 5368709120 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚29713458 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚210567130021888 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 UADeviceStatsMonitor.on_train_batch_start/r¥„«zequested_bytes.all.freed‚242270189813102 G>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4876 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚241563888549888 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 4654602240 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚62493061 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 2263326104064 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚50329600 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 122905600 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚243704440699392 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚4287 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚32779603 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚241563905654784 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚210567109378048 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚322 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚322 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚4287 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚243704440699392 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚61478912 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 4628521206 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚262 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 125829120 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 2140535044608 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚32779601 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 5494538240 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚262 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚27840307 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚209 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚41945088 KBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚4876 KDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚60 JBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚665 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 845178880 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚243704381649408 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚20643840 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚20408595 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚322 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17104896 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 2140493099520 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4289 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 5494538240 trainer/global_step‚8849 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚29709171 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 2140493099520 UHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚70973440 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚240134169001984 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚7431712 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚262 _step‚727 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 4654602240 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17104896 PIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚73×ÔB%2í- _wandbruntime‚2370  _runtime‚2370Þ®ËÇ&îÁM PDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚4295 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚262 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 2140766116352 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚322 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚41948160 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚20410800 NGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚65 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 2263520805376 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚61478912 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚223 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚242296664566678 H@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚665 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚210590177394688 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 5368709120 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚4293 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚29712458 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚41348068 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚32783154 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚243730994108928 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17104896 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚62499907 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 125829120 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚322 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 _step‚728 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚322 E5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5264 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 2263571131904 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 5494538240 EDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚27843351 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚241590227992576 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17104896 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚241590227992576 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 2136309771158 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 I@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚4876 G@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚67 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚240160337756160 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 5494538240 _runtime‚2370.686265634 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚41948160 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚212853677556224 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚62499907 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 DDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚62495612 G@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚60 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚32783154 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 845178880 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 122905600 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚241590210887680 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚210590156750848 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 IBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚60 % _timestamp‚1.7866610221557438e+09 K>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59053056 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚243730935055872 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 122905600 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚242296606179250 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚322 B9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚5264 D=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚60 trainer/global_step‚8849 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚7432616 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚212853748526592 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521206 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654602240 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚29716751 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 4552982528 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚50326528”d81Ì&ïÆM H@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚665 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚210590177394688 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 5368709120 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚4293 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚29712458 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚41348068 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚32783154 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚243730994108928 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17104896 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚62499907 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 125829120 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚322 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 _step‚728 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚322 E5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5264 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 2263571131904 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 5494538240 EDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚27843351 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚241590227992576 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17104896 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚241590227992576 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 2136309771158 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 I@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚4876 G@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚67 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚240160337756160 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 5494538240 _runtime‚2370.686265634 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚41948160 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚212853677556224 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚62499907 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 DDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚62495612 G@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚60 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚32783154 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 845178880 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 122905600 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚241590210887680 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚210590156750848 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 IBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚60 % _timestamp‚1.7866610221557438e+09 K>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59053056 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚243730935055872 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 122905600 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚242296606179250 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚322 B9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚5264 D=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚60 trainer/global_step‚8849 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚7432616 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚212853748526592 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521206 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654602240 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚29716751 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 4552982528 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚50326528 PDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚4295 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚262 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 2140766116352 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚322 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚41948160 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚20410800 NGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚65 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 2263520805376 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚61478912 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚223 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚242296664566678Øoü±Ã2ð- _wandbruntime‚2370  _runtime‚2370'¸?hrñì 0loss/FRt1/assignment_loss‚0.6668042581772435 /loss/FRt2/assignment_loss‚0.595075286459158 epoch‚22 trainer/global_step‚8849 % _timestamp‚1.7866610221569319e+09 _runtime‚2370.686754056 _step‚729 /loss/FRt1/detection_loss‚0.4874600883718998 0loss/FRt2/detection_loss‚0.48899841776752345 %loss/total_loss‚2.238338050775825# wòñ _runtime‚2370.686754056 _step‚729 /loss/FRt1/detection_loss‚0.4874600883718998 0loss/FRt2/detection_loss‚0.48899841776752345 %loss/total_loss‚2.238338050775825 0loss/FRt1/assignment_loss‚0.6668042581772435 /loss/FRt2/assignment_loss‚0.595075286459158 epoch‚22 trainer/global_step‚8849 % _timestamp‚1.7866610221569319e+09ÙZef2ó- _wandbruntime‚2382  _runtime‚2382Fð`ÇôÁ & lr-AdamW/pg1‚0.0009052673773228975 & lr-AdamW/pg2‚0.0009052673773228975 trainer/global_step‚8899 % _timestamp‚1.7866610341520414e+09 _runtime‚2382.683923928 _step‚730‚o² ÌõÆ & lr-AdamW/pg2‚0.0009052673773228975 trainer/global_step‚8899 % _timestamp‚1.7866610341520414e+09 _runtime‚2382.683923928 _step‚730 & lr-AdamW/pg1‚0.0009052673773228975ÚƖι2ö- _wandbruntime‚2382  _runtime‚2382S£Pp[÷·O ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚242880022544384 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚245032144354816 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17104896 PIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚65 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚41348020 KBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚4876 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 831258624 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚243591041300510 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 5368709120 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 122306526 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 125829120 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚4287 JBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚665 UHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚62584832 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 2275535107584 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚262 IBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚60 M@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚59049984 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚62835813 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚32957253 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚7472850 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚243591099687890 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚241443458678784 @7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚5264 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚245032085304832 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚242880022544384 IDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚242880005439488 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚213995075710464 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 5494538240 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚27991762 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚41945088 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚20643840 _runtime‚2382.684027233 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 2152079865344 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚32957251 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚429 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 5368709120 % _timestamp‚1.7866610341527112e+09 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚242880005439488 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚29874273 BDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4876 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚245032144354816 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚7472915 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚262 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚29874273 TGDeviceStatsMonitor.on_train_batch”gb_start/active_bytes.large_pool.current‚17104896 IDeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4289 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚29878560 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 845178880 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 5494538240 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 4552982528 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚322i‰žAÂ'ø¼O ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 2147599661086 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚41940992 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 4517085184 G>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4876 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚245032144354816 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚7472915 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚262 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚29874273 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17104896 IDeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4289 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚29878560 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 845178880 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 5494538240 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 4552982528 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚322 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚242880022544384 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚245032144354816 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17104896 PIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚65 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚41348020 KBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚4876 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 831258624 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚243591041300510 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 5368709120 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 122306526 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 125829120 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚4287 JBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚665 UHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚62584832 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 2275535107584 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚262 IBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚60 M@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚59049984 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚62835813 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚32957253 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚7472850 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚243591099687890 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚241443458678784 @7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚5264 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚245032085304832 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚242880022544384 IDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚242880005439488 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚213995075710464 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 5494538240 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚27991762 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚41945088 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚20643840 _runtime‚2382.684027233 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 2152079865344 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚32957251 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚429 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 5368709120 % _timestamp‚1.7866610341527112e+09 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚242880005439488 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚29874273 B5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5264 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚29877560 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚245058697764352 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 2152310934016 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚242906327777280 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 4517085184 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 I@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚4876 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 trainer/global_step‚8899 NADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚59053056 G@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚70 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚214018301662720 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚322 D=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚60 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 2152310934016 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚27994670 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚262 _step‚732 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 4552982528 B9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚5264 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚211742546034688 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚46132224 G@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚60 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚7473688 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 831258624 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 5368709120 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚322 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 4654602240 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚214018368438784 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 125829120 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚262 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚62838364 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚32960806 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚32960804 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 125829120 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚322 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚243617516054086 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 2147830233698 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚29877560 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 5494538240 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 125829120 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚223 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 5494538240 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚241469627432960 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654602240 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚209 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 2275776271872 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚322 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚4295 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚27994740 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 2147871581766 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚41348068 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 122306526 K>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59053056 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚243617457666658 % _timestamp‚1.7866610343156536e+09 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 122905600 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚62838364 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚62842659 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚29881853 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚7473620 SFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚66776064 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 IBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚60 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17104896 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17104896 EDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚62842659 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 5368709120 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 NADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚58387428 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 2152352882176 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 2152352882176 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 DDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚242906344882176 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚41948160 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 845178880 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚211742525390848 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521206 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚262 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚4293 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 4552982528 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 NGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚68 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 5368709120 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚429þÿÀ"ûÆM J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 4654602240 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚214018368438784 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 125829120 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚262 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚62838364 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚32960806 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚32960804 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 125829120 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚322 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚243617516054086 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 2147830233698 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚29877560 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 5494538240 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 125829120 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚223 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 5494538240 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚241469627432960 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654602240 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚209 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 2275776271872 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚322 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚4295 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚27994740 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 2147871581766 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚41348068 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 122306526 K>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59053056 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚243617457666658 % _timestamp‚1.7866610343156536e+09 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 122905600 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚62838364 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚62842659 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚29881853 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚7473620 SFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚66776064 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 IBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚60 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17104896 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17104896 EDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚62842659 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 5368709120 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 NADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚58387428 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 2152352882176 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 2152352882176 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 DDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚242906344882176 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚41948160 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 845178880 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚211742525390848 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521206 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚262 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚4293 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 4552982528 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 NGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚68 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 5368709120 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚429 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 P5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5264 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚29877560 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚245058697764352 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 2152310934016 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚242906327777280 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 4517085184 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 I@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚4876 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 trainer/global_step‚8899 NADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚59053056 G@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚70 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚214018301662720 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚322 D=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚60 XFDeviceStatsMonitor.on_train_batch_end/allocated_byte%Õ­ s.small_pool.freed‚ 2152310934016 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚27994670 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚262 _step‚732 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 4552982528 B9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚5264 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚211742546034688 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚46132224 G@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚60 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚7473688 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 831258624 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 5368709120 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚322Ü¿£èÝ2ü- _wandbruntime‚2382  _runtime‚2382a'4qýë 0loss/FRt1/assignment_loss‚0.6416854838283408 /loss/FRt2/assignment_loss‚0.614404526944535 /loss/FRt1/detection_loss‚0.5200644723205053 /loss/FRt2/detection_loss‚0.5178488165447447 epoch‚22 trainer/global_step‚8899 % _timestamp‚1.7866610343168433e+09 _runtime‚2382.846637052 %loss/total_loss‚2.294003299638126 _step‚733,F¡`vþð % _timestamp‚1.7866610343168433e+09 _runtime‚2382.846637052 %loss/total_loss‚2.294003299638126 _step‚733 0loss/FRt1/assignment_loss‚0.6416854838283408 /loss/FRt2/assignment_loss‚0.614404526944535 /loss/FRt1/detection_loss‚0.5200644723205053 /loss/FRt2/detection_loss‚0.5178488165447447 epoch‚22 trainer/global_step‚8899Ý2hµ•ÿ‚(:Š ¬‘ùÓ¯·†á gpu.0.gpu‚14.0 gpu.0.memory‚3, gpu.0.memoryAllocated‚7.0313262939453125) gpu.0.memoryAllocatedBytes‚ 6039863296 gpu.0.temp‚33.0 gpu.0.powerWatts‚186.504' gpu.0.enforcedPowerLimitWatts‚500.0 gpu.0.powerPercent‚37.3008 gpu.0.smClock‚1410 gpu.0.memoryClock‚1593$ gpu.0.correctedMemoryErrors‚1955# gpu.0.uncorrectedMemoryErrors‚0‘d“™€‚(:Ž ¬‘ùÓýŽÒã disk.drbd2165.out‚0 disk.dm-0.in‚43450.10546875 disk.sda3.in‚ 22492.453125$ disk.nvme0c0n1.in‚43450.10546875( disk.nvme0c0n1.out‚55738.87158203125& disk.nvme0n1.out‚55738.87158203125 network.sent‚ 1.1691221e+07 disk.drbd2165.in‚0 disk.sda.out‚ 45.2578125 disk.dm-1.out‚ 45.2578125$ memory_percent‚67.37360954284668( proc.memory.availableMB‚ 5345.5078125# proc.memory.rssMB‚ 2286.19921875 cpu‚27.133836805205718 disk.sda3.out‚ 45.2578125 disk.sda1.in‚0 disk.sda1.out‚0 disk.sda2.out‚0 disk.dm-1.in‚ 22492.453125 network.recv‚ 4.589098e+06) disk./.usagePercent‚32.69400819281455# disk.dm-0.out‚55738.87158203125 disk.sda.in‚ 22492.453125 disk.sda2.in‚0" disk.nvme0n1.in‚43450.10546875* proc.memory.percent‚13.953852653503418 proc.cpu.threads‚17$ disk./.usageGB‚487.1643486022949GúJa2- _wandbruntime‚2395  _runtime‚2395¯áÇ‚Á & lr-AdamW/pg1‚0.0008993819693442555 & lr-AdamW/pg2‚0.0008993819693442555 trainer/global_step‚8949 % _timestamp‚1.7866610467190442e+09 _runtime‚2395.250880174 _step‚734Em:ÌƒÆ & lr-AdamW/pg1‚0.0008993819693442555 & lr-AdamW/pg2‚0.0008993819693442555 trainer/global_step‚8949 % _timestamp‚1.7866610467190442e+09 _runtime‚2395.250880174 _step‚734ÞÛcá¾2„- _wandbruntime‚2395  _runtime‚2395ãƒG ½'…·O \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚20643840 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚262 G>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4876 JBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚665 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚429 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚212871867301888 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 F?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚60 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 2159202819714 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚48232448 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚4287 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚41945088 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 5368709120 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚262 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚322 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚244196139433984 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚212871846658048 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 2287800981504 BDeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4289 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚61478912 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚262 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 2163666631168 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 122905600 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 125829120 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 2163666631168 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚30043662 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚246359788960256 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17104896 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚20629095 _step‚735 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚244911892787918 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 5494538240 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 5368709120 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚33134903 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 4552982528 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚215159716515840 UHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚68876288 % _timestamp‚1.7866610467197058e+09 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 4517085184 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 2163708576256 D;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚5264 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚215159647639552 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 5494538240 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚322 KDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚60 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 2159161471694 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 IDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚246359848010240 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚33134901 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚322 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚242752731316224 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 122306526 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚244196139433984 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 4654602240 IDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚30043662 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚246359848010240 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17104896 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚244196122329088 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 2163708576256EÿÂ'†¼O WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 2163666631168 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚30043662 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚246359788960256 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17104896 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚20629095 _step‚735 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚244911892787918 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 5494538240 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 5368709120 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚33134903 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 4552982528 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚215159716515840 UHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚68876288 % _timestamp‚1.7866610467197058e+09 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 4517085184 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 2163708576256 D;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚5264 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚215159647639552 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 5494538240 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚322 KDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚60 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 2159161471694 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 IDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚246359848010240 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚33134901 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚322 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚242752731316224 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 122306526 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚244196139433984 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 4654602240 IDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚30043662 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚246359848010240 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17104896 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚244196122329088 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 2163708576256 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚20643840 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚262 G>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4876 JBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚665 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚429 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚212871867301888 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 F?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚60 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 2159202819714 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚48232448 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚4287 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚41945088 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 5368709120 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚262 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚322 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚244196139433984 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚212871846658048 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 2287800981504 BDeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4289 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚61478912 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚262 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 2163666631168 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 122905600 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 125829120߯xô2‡- _wandbruntime‚2395  _runtime‚2395>¢äŸ£!ˆÁM WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚244938367541494 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521206 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 5494538240 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚41948160 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚30042662 SFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚68873216 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚33138454 NADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚58387428 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚242778917109760 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 2163897699840 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚246386342366720 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚215182941083648 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 845178880 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚41348068 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚322 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚33138456 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17104896 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 2163939648000 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚33138454 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 2288095282176 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 2159392044306 D8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 125829120 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚262 EDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚28145365 % _timestamp‚1.7866610468812008e+09 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚28145438 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚429 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 122306526 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 P6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚322 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 4517085184 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 5494538240 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 5494538240 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 125829120 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚244222461771776 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚244222444666880 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚63185411 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚33138456 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚20643840 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 5368709120 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 H@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚665 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 5368709120 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚322 _step‚736 >5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5264 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚4293 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17104896 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 2288047052800 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚262 trainer/global_step‚8949 K>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59053056 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 2163897699840 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 125829120 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654602240 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 C6DeviceStatsMonitor.on_train_batch_end/active.ÑJ ý$all.freed‚63181116 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚7514136 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚61478912 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 2159433392374 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚262 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚246386401419776 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 4552982528 _runtime‚2395.411467044 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 IBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚60 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚30046955 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚246386401419776 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚322 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 4552982528 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚212894894030848ÛõÔ}Ì&‰ÆM G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚63185411 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚28145365 % _timestamp‚1.7866610468812008e+09 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚28145438 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚429 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 122306526 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 P6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚322 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 4517085184 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 5494538240 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 5494538240 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 125829120 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚244222461771776 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚244222444666880 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚63185411 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚33138456 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚20643840 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 5368709120 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 H@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚665 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 5368709120 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚322 _step‚736 >5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5264 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚4293 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17104896 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 2288047052800 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚262 trainer/global_step‚8949 K>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59053056 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 2163897699840 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 125829120 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654602240 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚63181116 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚7514136 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚61478912 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 2159433392374 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚262 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚246386401419776 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 4552982528 _runtime‚2395.411467044 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 IBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚60 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚30046955 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚246386401419776 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚322 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 4552982528 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚212894894030848 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚244938367541494 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521206 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 5494538240 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚41948160 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚30042662 SFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚68873216 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚33138454 NADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚58387428 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚242778917109760 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 2163897699840 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚246386342366720 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚215182941083648 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 845178880 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚41348068 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚322 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚33138456 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17104896 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 2163939648000 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚33138454 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 2288095282176 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 2159392044306 D8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 125829120 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚262 EDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 4628521206 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 5368709120 IBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚60 JBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚665 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚322 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚4287 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚216324290804736 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚246232802662706 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 2170764630322 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 2170723282302 IDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 4552982528 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚245512239218688 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚63521317 IDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚247687492615680 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚245512256323584 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚28293662 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 845178880 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚61478912 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 @7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚5264 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 4552982528 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 4654602240 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚41945088 PCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚58387380 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚223 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 5494538240 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚262 _runtime‚2407.517858114 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚7554315 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 trainer/global_step‚8999 % _timestamp‚1.7866610589842517e+09 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚247687551665664 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 2300127933440 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚322 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 5368709120 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 G>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4876 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 2175295342080 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚322 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚246232744275326 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚30204477 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚247687492615680 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚30208764 IBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚76 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚216324363875328 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚244062038032384 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚41348020 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚214024235941888 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 125829120 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 4654602240 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4289 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17104896 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚209 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚52426752 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚262 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚262 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0ÝØÒ²¯!”¼O XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 5368709120 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚30204477 BDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 4628521206 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 5368709120 IBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚60 JBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚665 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚322 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚4287 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚216324290804736 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚246232802662706 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 2170764630322 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 2170723282302 IDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 4552982528 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚245512239218688 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚63521317 IDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚247687492615680 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚245512256323584 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚28293662 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 845178880 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚61478912 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 @7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚5264 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 4552982528 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 4654602240 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚41945088 PCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚58387380 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚223 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 5494538240 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚262 _runtime‚2407.517858114 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚7554315 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 trainer/global_step‚8999 % _timestamp‚1.7866610589842517e+09 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚247687551665664 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 2300127933440 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚322 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 5368709120 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 G>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4876 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 2175295342080 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚322 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚246232744275326 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚30204477 UADeviceStatsMonitor.on_train_batch_start/allo1,Ëcated_bytes.all.freed‚247687492615680 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚30208764 IBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚76 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚216324363875328 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚244062038032384 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚41348020 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚214024235941888 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 125829120 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 4654602240 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4289 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17104896 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚209 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚52426752 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚262 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚262 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚33312553 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚28293586ãqžº†2•- _wandbruntime‚2407  _runtime‚2407”1jÌÇ&–ÁM T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚247714105075200 B9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚5264 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 831258624 BDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚63528163 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚30212057 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 2170953854914 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 125829120 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚429 IBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚60 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 5368709120 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚247714046022144 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚245538578661376 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚7555156 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚7555089 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 122306526 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 2175484465664 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚33316106 EDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654602240 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 2175526413824 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 125829120 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚63523868 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚247714105075200 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚41948160 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚214047262670848 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 2175484465664 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 5368709120 >5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5264 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 122905600 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 2175526413824 trainer/global_step‚8999 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚41948160 _step‚740 P6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚322 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 4654602240 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17104896 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 2300326304768 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 122905600 NADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚59053056 K>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59053056 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚245538578661376 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚20741552 NGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚67 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚214047283314688 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚244088223825920 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 2170995202982 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚4295 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚4293 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚30207764 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 5494538240 DDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521206 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚262 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 4517085184 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 5368709120 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚33316104 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17104896 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚244088206786560 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚262 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 4552982528 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚28296639 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚223 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 D=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚60 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚20741550 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚246259219028902 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 5494538240 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚4293 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 4552982528 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚209 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 I@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚4876 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚20643840 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚41348068 G@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚69 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚216347653654528 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 E6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚322 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚322 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚322 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚30212057 H@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚665 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚44035072 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚262 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚33316106 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚33316104 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚61478912©CÏÌ&—ÆM K>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59053056 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚245538578661376 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚20741552 NGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚67 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚214047283314688 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚244088223825920 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 2170995202982 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚4295 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚4293 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚30207764 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 5494538240 DDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521206 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚262 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 4517085184 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 5368709120 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚33316104 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17104896 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚244088206786560 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚262 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 4552982528 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚28296639 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚223 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 D=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚60 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚20741550 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚246259219028902 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 5494538240 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚4293 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 4552982528 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚209 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 I@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚4876 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚20643840 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚41348068 G@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚69 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚216347653654528 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 E6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚322 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚322 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚322 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚30212057 H@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚665 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚44035072 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚262 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚33316106 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚33316104 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚61478912 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚247714105075200 B9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚5264 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 831258624 BDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚63528163 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚30212057 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 2170953854914 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 125829120 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚429 IBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚60 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 5368709120 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚247714046022144 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚245538578661376 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚7555156 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚7555089 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 122306526 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 2175484465664 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚33316106 EDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654602240 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 2175526413824 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 125829120 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚63523868 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚247714105075200 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚41948160 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚214047262670848 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 2175484465664 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 5368709120 >5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5264 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 122905600 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 2175526413824 trainer/global_step‚8999 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚41948160 _step‚740 P6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚322 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 4654602240 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17104896 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 2300326304768 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 122905600 NADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚59053056äójöø2˜- _wandbruntime‚2407  _runtime‚2407FVp™ê /loss/FRt1/detection_loss‚0.5065305717032571 /loss/FRt2/detection_loss‚0.4765461591335837 epoch‚22 trainer/global_step‚8999 % _timestamp‚1.7866610591471846e+09 _runtime‚2407.677041983 $loss/total_loss‚2.25679934071845 _step‚741 /loss/FRt1/assignment_loss‚0.681105790748461 0loss/FRt2/assignment_loss‚0.5926168191331481U[oušï /loss/FRt2/detection_loss‚0.4765461591335837 epoch‚22 trainer/global_step‚8999 % _timestamp‚1.7866610591471846e+09 _runtime‚2407.677041983 $loss/total_loss‚2.25679934071845 _step‚741 /loss/FRt1/assignment_loss‚0.681105790748461 0loss/FRt2/assignment_loss‚0.5926168191331481 /loss/FRt1/detection_loss‚0.5065305717032571åÓY‹–›‚(:‹ Ê‘ùÓóÊúà gpu.0.gpu‚64.0 gpu.0.memory‚12, gpu.0.memoryAllocated‚7.0313262939453125) gpu.0.memoryAllocatedBytes‚ 6039863296 gpu.0.temp‚35.0 gpu.0.powerWatts‚212.563' gpu.0.enforcedPowerLimitWatts‚500.0 gpu.0.powerPercent‚42.5126 gpu.0.smClock‚1410 gpu.0.memoryClock‚1593$ gpu.0.correctedMemoryErrors‚1955# gpu.0.uncorrectedMemoryErrors‚0n»›e”œ‚(:‰ Ê‘ùÓÊ÷°ã& disk.nvme0n1.out‚55748.08935546875# disk.dm-0.out‚55748.08935546875 disk.sda.in‚23471.75390625 disk.sda.out‚45.6875 network.recv‚ 4.629155e+06 disk.sda1.in‚0 disk.dm-0.in‚43458.08984375 disk.sda3.in‚23471.75390625# memory_percent‚67.3727035522461 cpu‚27.133938482959287 disk.drbd2165.out‚0 network.sent‚ 1.1838458e+07 disk.sda2.in‚0 disk.dm-1.in‚23471.75390625 disk.dm-1.out‚45.6875( disk.nvme0c0n1.out‚55748.08935546875# proc.memory.rssMB‚ 2286.19921875* proc.memory.percent‚13.953852653503418) disk./.usagePercent‚32.69400051258853% disk./.usageGB‚487.16423416137695 disk.drbd2165.in‚0 disk.sda3.out‚45.6875$ disk.nvme0c0n1.in‚43458.08984375& proc.memory.availableMB‚ 5345.65625 proc.cpu.threads‚17 disk.sda1.out‚0 disk.sda2.out‚0" disk.nvme0n1.in‚43458.08984375Ð'Ü_2- _wandbruntime‚2419  _runtime‚2419£äÆÇžÁ & lr-AdamW/pg1‚0.0008875827484377221 & lr-AdamW/pg2‚0.0008875827484377221 trainer/global_step‚9049 % _timestamp‚1.7866610713656766e+09 _runtime‚2419.895818802 _step‚742óÌŸÆ _runtime‚2419.895818802 _step‚742 & lr-AdamW/pg1‚0.0008875827484377221 & lr-AdamW/pg2‚0.0008875827484377221 trainer/global_step‚9049 % _timestamp‚1.7866610713656766e+09æ”±!S2 - _wandbruntime‚2419  _runtime‚2419Ü0Ž!¡¶O NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚209 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚41348020 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 4552982528 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚4287 D;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚5264 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚30369579 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17104896 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚33490201 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 5368709120 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 831258624 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚245371310669824 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 F?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚60 PCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚59049984 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚249015196271104 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 4654602240 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚30369579 KBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚4876 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 2186840162816 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4289 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚33490203 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 4654602240 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚20849595 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚245371327709184 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 122905600 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚7595330 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚215176604581888 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 2182326440930 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚322 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚20849597 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 2186840162816 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚223 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 122306526 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚262 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 2186882107904 IDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 JBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚665 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚7595264 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 2312409158144 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚246828373213184 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚246828356108288 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 2312363022848 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚30373866 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚217488946960896 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚61478912 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 125829120 G>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4876 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚249015196271104 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚41945088 PCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚58387380 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 125829120 _step‚743 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚429 KDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚60 IBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚60 @7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚5264 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚246828373213184 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 4552982528 UHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚66779136 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚262 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚249015255321088 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚246828356108288 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚322 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 5494538240 ^JDeviceStatsMonitor.on_train_batch_start/inaÄ]· .ctive_split_bytes.all.allocated‚217489013740032 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 4517085184 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 5368709120 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚262 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚20643840 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚46135296 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 _runtime‚2419.899873598 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚28444927 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 125829120 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚322 IDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 JBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚665 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚7595264 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 2312409158144 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚246828373213184 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚246828356108288 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 2312363022848 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚30373866 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚217488946960896 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚61478912 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 125829120 G>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4876 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚249015196271104 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚41945088 PCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚58387380 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 125829120 _step‚743 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚429 KDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚60 IBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚60 @7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚5264 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚246828373213184 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 4552982528 UHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚66779136 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚262 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚249015255321088 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚246828356108288 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚322 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 5494538240 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚217489013740032 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 4517085184 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 5368709120 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚262 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚20643840 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚46135296 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 _runtime‚2419.899873598 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚28444927 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 125829120 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚322 IDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚249015196271104 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 4654602240 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚30369579 KBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚4876 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 2186840162816 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4289 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚33490203 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 4654602240 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚20849595 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚245371327709184 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 122905600 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚7595330 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚215176604581888 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 2182326440930 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚322 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚20849597 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 2186840162816 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚223 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 122306526çBëÈ2£- _wandbruntime‚2420  _runtime‚2420‹-‚Æ&¤ÀM \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚217512307516928 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 845178880 E6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚322 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚41948160 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 831258624 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 5368709120 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 EDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59053056 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚246854678446080 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚322 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 2312655562240 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 2187071231488 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚215199631310848 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚247580012128882 trainer/global_step‚9049 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 122905600 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 122905600 SFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚66776064 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚262 $ _timestamp‚1.786661071535364e+09 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 4552982528 B9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚5264 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚20851802 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚33493754 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚245397496463360 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 5368709120 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 125829120 G@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚60 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚249041808730624 NGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚72 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚247580070516310 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 IBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚60 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚4295 I@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚4876 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚28447867 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚215199651954688 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 2182515665522 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚262 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚30372866 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17104896 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚30377159 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚30372866 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 2312609430016 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚61478912 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚322 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚41348068 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚28447941 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚429 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚41948160 D=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚60 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521206 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 122306526 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 2187071231488 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 5494538240 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚63870915 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚63866620 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚33493756 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚46132224 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚63866620 BDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654602240 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚33493754 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚63870915 NADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚59053056 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚20643840 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 5494538240 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚249041808730624 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 NADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚58387428 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚4293 >5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5264 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚246854678446080 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚217512240740864 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚262 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚30377159 _runtime‚2420.066034002 _step‚744 PDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚30372866 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17104896 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚30377159 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚30372866 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 2312609430016 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚61478912 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚322 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚41348068 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚28447941 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚429 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚41948160 D=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚60 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521206 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 122306526 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 2187071231488 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 5494538240 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚63870915 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚63866620 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚33493756 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚46132224 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚63866620 BDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654602240 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚33493754 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚63870915 NADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚59053056 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚20643840 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 5494538240 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚249041808730624 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 NADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚58387428 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚4293 >5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5264 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚246854678446080 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚217512240740864 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚262 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚30377159 _runtime‚2420.066034002 _step‚744 P6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚322 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚41948160 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 831258624 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 5368709120 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 EDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59053056 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚246854678446080 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚322 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 2312655562240 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 2187071231488 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚215199631310848 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚247580012128882 trainer/global_step‚9049 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 122905600 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 122905600 SFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚66776064 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚262 $ _timestamp‚1.786661071535364e+09 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 4552982528 B9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚5264 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚20851802 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚33493754 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚245397496463360 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 5368709120 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 125829120 G@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚60 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚249041808730624 NGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚72 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚247580070516310 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 IBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚60 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚4295 I@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚4876 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚28447867è Û@2¦- _wandbruntime‚2420  _runtime‚2420Èyßîn§è 0loss/FRt1/assignment_loss‚0.6445954857390763 epoch‚22 trainer/global_step‚9049 _runtime‚2420.066463263 _step‚745 /loss/FRt2/assignment_loss‚0.611377319873321 /loss/FRt1/detection_loss‚0.5184256618949592 /loss/FRt2/detection_loss‚0.5020266184812191 %loss/total_loss‚2.276425085988576 " _timestamp‚1.7866610715366e+09Ç<° s¨í /loss/FRt1/detection_loss‚0.5184256618949592 /loss/FRt2/detection_loss‚0.5020266184812191 %loss/total_loss‚2.276425085988576 " _timestamp‚1.7866610715366e+09 0loss/FRt1/assignment_loss‚0.6445954857390763 epoch‚22 trainer/global_step‚9049 _runtime‚2420.066463263 _step‚745 /loss/FRt2/assignment_loss‚0.611377319873321é³B€Vü©‚(:ˆ Ù‘ùӆ؟á gpu.0.gpu‚1.0 gpu.0.memory‚0, gpu.0.memoryAllocated‚7.0313262939453125) gpu.0.memoryAllocatedBytes‚ 6039863296 gpu.0.temp‚34.0 gpu.0.powerWatts‚218.485' gpu.0.enforcedPowerLimitWatts‚500.0 gpuîfT—.0.powerPercent‚43.697 gpu.0.smClock‚1410 gpu.0.memoryClock‚1593$ gpu.0.correctedMemoryErrors‚1955# gpu.0.uncorrectedMemoryErrors‚0Éϛҗª‚(:Œ Ù‘ùÓÀ¡Äã& disk.nvme0c0n1.out‚55751.990234375 proc.cpu.threads‚17 disk.dm-1.out‚ 45.8359375 disk.drbd2165.out‚0! disk.dm-0.out‚55751.990234375 disk.sda.out‚ 45.8359375# proc.memory.rssMB‚ 2286.19921875 disk.sda2.out‚0 disk.sda3.out‚ 45.8359375$ disk.nvme0c0n1.in‚43458.08984375 disk.sda.in‚23471.75390625# memory_percent‚67.3729419708252( proc.memory.availableMB‚ 5345.6171875* proc.memory.percent‚13.953852653503418$ disk./.usageGB‚487.1602096557617 disk.sda1.out‚0 disk.sda2.in‚0" disk.nvme0n1.in‚43458.08984375$ disk.nvme0n1.out‚55751.990234375 disk.drbd2165.in‚0 disk.dm-0.in‚43458.08984375 cpu‚27.133728624635424 network.sent‚ 1.1913595e+07 network.recv‚ 4.662235e+06( disk./.usagePercent‚32.6937304246399 disk.sda1.in‚0 disk.dm-1.in‚23471.75390625 disk.sda3.in‚23471.75390625ñŠ Ë2«- _wandbruntime‚2432  _runtime‚2432ãS-ƬÀ _runtime‚2432.437827906 _step‚746 & lr-AdamW/pg1‚0.0008816696933517412 & lr-AdamW/pg2‚0.0008816696933517412 trainer/global_step‚9099 $ _timestamp‚1.786661083907777e+09´ã8Ë­Å & lr-AdamW/pg1‚0.0008816696933517412 & lr-AdamW/pg2‚0.0008816696933517412 trainer/global_step‚9099 $ _timestamp‚1.786661083907777e+09 _runtime‚2432.437827906 _step‚746êm¦2®- _wandbruntime‚2432  _runtime‚2432ÞÍÚN½'¯·O NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚4287 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚223 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 2193888251538 JBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚665 UHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚62584832 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 5368709120 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚33667851 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚250342958976512 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚33667851 G>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4876 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚30538968 PCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚59049984 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚216328973221888 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚61478912 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 4552982528 D;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚5264 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚20959847 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚20643840 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚246680617385984 trainer/global_step‚9099 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17104896 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 4517085184 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚250342958976512 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 122905600 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚262 _runtime‚2432.442861711 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚250342899926528 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 2324661526016 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚30534681 M@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚59049984 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 125829120 KDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚60 _step‚747 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚41945088 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚28595633 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚248144490102784 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚41940992 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 4628521206 IBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚60 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚33667853 IBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚67 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚7635721 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚248874505637522 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 4552982528 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 2198468873728 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚250342899926528 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17104896 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚218653614104064 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚248874447250142 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 @7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚5264 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚7635786 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚218653676688896 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 2193846903518 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 5494538240 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚33667853 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 2198426928640 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚429 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 845178880 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 5368709120 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚30534681 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚28595566 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚322 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 125829120 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚248144490102784 BDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚4287 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚248144472997888 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 2198426928640 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚216328952578048 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚41348020 KBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚4876 PIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚65 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 831258624 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 4654602240 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 2198468873728 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 E8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚64202532 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚41945088 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚209 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 2324703467008 PCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚58387380 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚246680600346624 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚20959845 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 5494538240 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚64206821 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 5368709120 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 125829120 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚262 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 F?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚60 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 4654602240 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4289 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 5494538240 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚262 IDeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4289 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 5494538240 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚262 IDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4876 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚30538968 PCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚59049984 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚216328973221888 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚61478912 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 4552982528 D;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚5264 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚20959847 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚20643840 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚246680617385984 trainer/global_step‚9099 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17104896 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 4517085184 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚250342958976512 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 122905600 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚262 _runtime‚2432.442861711 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚250342899926528 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 2324661526016 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚30534681 M@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚59049984 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 125829120 KDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚60 _step‚747 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚41945088 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚28595633 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚248144490102784 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚41940992 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 4628521206 IBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚60 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚33667853 IBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚67 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚7635721 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚248874505637522 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 4552982528 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 2198468873728 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚250342899926528 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17104896 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚218653614104064 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚248874447250142 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 @7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚5264 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚7635786 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚218653676688896 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 2193846903518 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 5494538240 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚33667853 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 2198426928640 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚429 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 845178880 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 5368709120 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚30534681 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚28595566 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚322 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 125829120 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚248144490102784 BDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚4287 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚248144472997888ëõc”72±- _wandbruntime‚2432  _runtime‚2432¤ÒåÇ&²ÁM aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 2324948822528 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚246706803179520 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 4552982528 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 122905600 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚322 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚33671404 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17104896 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚33671406 BDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚322 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654602240 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚4293 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521206 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 125829120 _step‚748 >5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5264 K>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59053056 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚248170795335680 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 G@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚67 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 5368709120 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17104896 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 2198699945472 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚33671404 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚218676969417216 SFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚66776064 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚46132224 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 4654602240 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚30537968 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 NADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚58387428 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 4517085184 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚41348068 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 125829120 D=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚60 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 5368709120 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚248170812440576 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 2198699945472 B9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚5264 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚246706786140160 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 2198657997312 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚41948160 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚216352020594688 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 125829120 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚30542261 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚7636581 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚248900922003718 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 5494538240 G@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚60 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚4295 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚250369453332992 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 2198657997312 NGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚65 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚30542261 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚20962052 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚216351999950848 trainer/global_step‚9099 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚28598633 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚20962050 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚262 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚250369512386048 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 831258624 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 2194077476130 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚30537968 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚322 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚250369512386048 H@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚665 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚20643840 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚262 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚223 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 4552982528 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 2324902690304 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 2194118824198 % _timestamp‚1.7866610840809946e+09 _runtime‚2432.611325444 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚64213667 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚248170795335680 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚4293 I@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚4876 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 EDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚28598566 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚209 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚218676902641152 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚61478912 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 D6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚322 NADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚59053056 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚33671406 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 122905600 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚64213667 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚429 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 5368709120 IBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚60 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 122306526 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚248170812440576 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚248900863616290 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚41948160ÜýÿT³ÆM C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚322 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚250369512386048 H@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚665 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚20643840 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚262 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚223 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 4552982528 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 2324902690304 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.$*é7´#small_pool.allocated‚ 2194118824198 % _timestamp‚1.7866610840809946e+09 _runtime‚2432.611325444 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚64213667 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚248170795335680 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚4293 I@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚4876 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 EDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚28598566 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚209 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚218676902641152 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚61478912 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 D6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚322 NADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚59053056 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚33671406 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 122905600 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚64213667 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚429 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 5368709120 IBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚60 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 122306526 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚248170812440576 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚248900863616290 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚41948160 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 2324948822528 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚246706803179520 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 4552982528 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 122905600 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚322 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚33671404 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17104896 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚33671406 BDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚322 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654602240 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚4293 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521206 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 125829120 _step‚748 >5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5264 K>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59053056 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚248170795335680 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 G@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚67 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 5368709120 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17104896 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 2198699945472 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚33671404 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚218676969417216 SFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚66776064 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚46132224 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 4654602240 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚30537968 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 NADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚58387428 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 4517085184 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚41348068 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 125829120 D=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚60 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 5368709120 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚248170812440576 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 2198699945472 B9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚5264 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚246706786140160 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 2198657997312 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚41948160 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚216352020594688 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 125829120 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚30542261 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚7636581 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚248900922003718 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 5494538240 G@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚60 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚4295 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚250369453332992 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 2198657997312 NGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚65 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚30542261 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚20962052 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚216351999950848 trainer/global_step‚9099 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚28598633 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚20962050 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚262 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚250369512386048 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 831258624 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 2194077476130 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚30537968 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0ìiú?è2´- _wandbruntime‚2432  _runtime‚2432¶Ñ@qµë epoch‚22 trainer/global_step‚9099 _runtime‚2432.611945274 _step‚749 0loss/FRt1/assignment_loss‚0.6013775933640284 .loss/FRt1/detection_loss‚0.473610290179454 %loss/total_loss‚2.050715017058078 % _timestamp‚1.7866610840820963e+09 /loss/FRt2/assignment_loss‚0.519192863124769 0loss/FRt2/detection_loss‚0.45653427038982614ÿv¶ð 0loss/FRt1/assignment_loss‚0.6013775933640284 .loss/FRt1/detection_loss‚0.473610290179454 %loss/total_loss‚2.050715017058078 % _timestamp‚1.7866610840820963e+09 /loss/FRt2/assignment_loss‚0.519192863124769 0loss/FRt2/detection_loss‚0.45653427038982614 epoch‚22 trainer/global_step‚9099 _runtime‚2432.611945274 _step‚749íb¥@é2·- _wandbruntime‚2444  _runtime‚2444<·äǸÁ _runtime‚2444.940883257 _step‚750 & lr-AdamW/pg1‚0.0008757481813674013 & lr-AdamW/pg2‚0.0008757481813674013 trainer/global_step‚9149 % _timestamp‚1.7866610964088042e+09ÑÎ×ÔÌ¹Æ _step‚750 & lr-AdamW/pg1‚0.0008757481813674013 & lr-AdamW/pg2‚0.0008757481813674013 trainer/global_step‚9149 % _timestamp‚1.7866610964088042e+09 _runtime‚2444.940883257îàQ —2º- _wandbruntime‚2444  _runtime‚2444³êg¼'»¶O TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚30704070 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 4517085184 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚249460589887488 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚33845501 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚322 KDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚60 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚249460606992384 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 4552982528 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚251670603581952 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚249460589887488 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚429 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 _step‚751 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚217481341861888 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 2336933893632 BDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚41348020 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 122905600 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚33845503 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚247989890023424 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 845178880 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 5368709120 G>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4876 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚249460606992384 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚61478912 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 2210013694464 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚20643840 E8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚64545284 @7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚5264 IBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚60 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 4628521206 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 5494538240 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚251670603581952 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚64549573 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚223 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 2210055639552 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 2210013694464 PIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚58 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚250195298737550 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 122306526 KBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚4876 PCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚58387380 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚262 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚4287 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 4654602240 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚41945088 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚209 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 2205450062146 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 125829120 M@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚59049984 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 5494538240 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 5368709120 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 125829120 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚322 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 _runtime‚2444.940967666 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚247989907062784 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 5494538240 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 $ _timestamp‚1.786661096409469e+09 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚7676222 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚28746319 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚251670662631936 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚4287 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17104896 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 122905600 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚41945088 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 F>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚7676164 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚219818255111680 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 4654602240 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚262 IBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚60 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚21070097 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚21070095 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 trainer/global_step‚9149 C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚4289 D;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚5264 JBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚665 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚33845501 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚217481321218048 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 125829120 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 2210055639552 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚30699783 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚28746259 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚33845503 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚30704070 PCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚59049984 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 831258624 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚322 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚262 IDeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4289 UHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚58390528 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚37746688 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 2205408714126 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 F?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚60 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 4552982528 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚30699783 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚251670662631936 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚322¶4ÌÁ'¼»O GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 F?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚60 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 4552982528 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚30699783 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚251670662631936 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚322 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚30704070 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 4517085184 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚249460589887488 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚33845501 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚322 KDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚60 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚249460606992384 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 4552982528 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚251670603581952 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚249460589887488 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚429 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 _step‚751 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚217481341861888 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 2336933893632 BDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚41348020 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 122905600 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚33845503 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚247989890023424 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 845178880 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 5368709120 G>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4876 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚249460606992384 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚61478912 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 2210013694464 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚20643840 E8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚64545284 @7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚5264 IBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚60 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 4628521206 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 5494538240 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚251670603581952 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚64549573 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚223 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 2210055639552 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 2210013694464 PIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚58 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚250195298737550 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 122306526 KBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚4876 PCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚58387380 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚262 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚4287 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 4654602240 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚41945088 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚209 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 2205450062146 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 125829120 M@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚59049984 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 5494538240 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 5368709120 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 125829120 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚322 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 _runtime‚2444.940967666 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚247989907062784 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 5494538240 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 $ _timestamp‚1.786661096409469e+09 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚7676222 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚28746319 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚251670662631936 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚4287 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17104896 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 122905600 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚41945088 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 F>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚7676164 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚219818255111680 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 4654602240 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚262 IBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚60 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚21070097 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚21070095 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 trainer/global_step‚9149 C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚4289 D;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚5264 JBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚665 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚33845501 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚217481321218048 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 125829120 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 2210055639552 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚30699783 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚28746259 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚33845503 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚30704070 PCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚59049984 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 831258624 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚322 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚262 IDeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4289 UHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚58390528 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚37746688 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 2205408714126ïMt¤F–½‚(:‹ è‘ùÓ¢Þ¤á gpu.0.gpu‚64.0 gpu.0.memory‚12, gpu.0.memoryAllocated‚7.0313262939453125) gpu.0.memoryAllocatedBytes‚ 6039863296 gpu.0.temp‚34.0 gpu.0.powerWatts‚200.209' gpu.0.enforcedPowerLimitWatts‚500.0 gpu.0.powerPercent‚40.0418 gpu.0.smClock‚1410 gpu.0.memoryClock‚1593$ gpu.0.correctedMemoryErrors‚1955# gpu.0.uncorrectedMemoryErrors‚0Gqx›¾‚(: è‘ùÓŸýöâ disk.sda3.out‚ 46.07421875 disk.sda2.out‚0 disk.dm-1.in‚23471.75390625 disk.dm-1.out‚ 46.07421875# memory_percent‚67.3732042312622 cpu‚27.13374019399739 network.recv‚ 4.68574e+06 disk.sda1.in‚0 disk.sda2.in‚0! disk.nvme0n1.in‚ 43458.1953125% disk.nvme0n1.out‚55759.3564453125 disk.drbd2165.in‚0 disk.drbd2165.out‚0) proc.memory.availableMB‚ 5345.57421875 network.sent‚ 1.1979851e+07% disk./.usageGB‚487.16033935546875# disk.nvme0c0n1.in‚ 43458.1953125 disk.dm-0.in‚ 43458.1953125" disk.dm-0.out‚55759.3564453125# proc.memory.rssMB‚ 2286.19921875* proc.memory.percent‚13.953852653503418) disk./.usagePercent‚32.69373912889606 disk.sda3.in‚23471.75390625' disk.nvme0c0n1.out‚55759.3564453125 disk.sda.in‚23471.75390625 disk.sda.out‚ 46.07421875 disk.sda1.out‚0 proc.cpu.threads‚17¬ÃÇZ2¿- _wandbruntime‚2445  _runtime‚2445ýŸÙ¿úÀÀM QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 845178880 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 4654602240 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚64556419 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚262 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚30707363 SFDeviceStatsMonitor.on_train_batch_end/inactive˜†Ì$_split_bytes.all.current‚62581760 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 2205639286738 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 5368709120 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚322 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 >5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5264 DDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚28749262 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 2337178727936 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 trainer/global_step‚9149 $ _timestamp‚1.786661096571869e+09 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚4295 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 2210286711296 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚21072300 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚322 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚64552124 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 2210286711296 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚41948160 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654602240 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 2210244763136 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 4552982528 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 125829120 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚262 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚64556419 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚30703070 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚251697156988416 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚41348068 NADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚59053056 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚33849054 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 _runtime‚2445.102120772 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 122905600 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 5368709120 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 4552982528 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 2210244763136 EDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521206 K>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59053056 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚249486912225280 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚250221773491126 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚249486929330176 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚30707363 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚219841609900544 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚4293 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17104896 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 BDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚64556419 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚262 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚30707363 SFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚62581760 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 2205639286738 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 5368709120 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚322 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 >5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5264 DDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚28749262 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 2337178727936 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 trainer/global_step‚9149 $ _timestamp‚1.786661096571869e+09 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚4295 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 2210286711296 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚21072300 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚322 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚64552124 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 2210286711296 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚41948160 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654602240 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 2210244763136 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 4552982528 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 125829120 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚262 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚64556419 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚30703070 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚251697156988416 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚41348068 NADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚59053056 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚33849054 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 _runtime‚2445.102120772 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 122905600 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 5368709120 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 4552982528 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 2210244763136 EDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521206 K>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59053056 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚249486912225280 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚250221773491126 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚249486929330176 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚30707363 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚219841609900544 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚4293 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17104896 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 BDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4876 IBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚66 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 2349232535040 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚322 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚252998307237376 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚250776723881984 JBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚665 PCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚59049984 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚41945088 PIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚64 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚34023151 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚223 F?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚60 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 2221642405376 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 2221642405376 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 2221600460288 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 122905600 KBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚4876 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 4517085184 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 4552982528 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 122905600 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4289 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚30864885 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚209 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚220982922393088 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚251516150224958 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 125829120 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 5494538240 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 125829120 KDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚60 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚21180345 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 831258624 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 2217011872754 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚39843840 trainer/global_step‚9199 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚41945088 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 4552982528 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚61478912 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚249299196739584 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 122306526 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 5368709120 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚34023153 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚28897188 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 5494538240 IDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚250776706777088 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚30869172 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚218633689858048 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚262 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚34023151 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚4287 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 @7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚5264 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17104896 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚250776723881984 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚28897122 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚21180347 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚322 E8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚64888036 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 2221600460288)_7ASʺO M@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚59049984 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚218633710501888 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 5368709120 F>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚250776706777088 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚30869172 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚218633689858048 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚262 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚34023151 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚4287 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 @7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚5264 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17104896 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚250776723881984 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚28897122 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚21180347 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚322 E8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚64888036 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 2221600460288 BDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4876 IBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚66 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 2349232535040 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚322 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚252998307237376 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚250776723881984 JBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚665 PCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚59049984 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚41945088 PIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚64 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚34023151 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚223 F?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚60 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 2221642405376 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 2221642405376 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 2221600460288 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 122905600 KBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚4876 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 4517085184 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 4552982528 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 122905600 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4289 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚30864885 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚209 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚220982922393088 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚251516150224958 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 125829120 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 5494538240 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 125829120 KDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚60 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚21180345 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 831258624 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 2217011872754 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚39843840 trainer/global_step‚9199 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚41945088 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 4552982528 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚61478912 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚249299196739584 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 122306526 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 5368709120 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚34023153 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚28897188 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 5494538240 IDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚28900123 SFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚66776064 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 831258624 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 2217242445414 _runtime‚2457.571691492 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚30872465 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚30872465 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 NGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚63 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 125829120 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 2221873477120 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚4293 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚221006210930176 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 2349473699328 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 5368709120 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚4295 EDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 2221873477120 B9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚5264 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 845178880 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 NADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚58387428 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 125829120 _step‚756 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17104896 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 5494538240 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚253024919696896 NADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚59053056 H@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚665 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚28900188 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚429 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚322 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚34026706 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚41948160 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚221006277706240 IBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚60 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 2221831528960 B8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚262 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 2221831528960 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚250803029114880 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 5368709120 G@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚60 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚322 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚41348068 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚30868172 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 122905600 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚218656757874688 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17104896 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚253024860643840 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 4517085184 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚250803046219776 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚218656737230848 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 trainer/global_step‚9199 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚64899171 K>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59053056 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚34026704 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚251542624978534 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 2349519831552 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚262 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 D6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚322 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 5494538240 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚223 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚64894876 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚21182552 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚20643840 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521206 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚250803029114880 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 122905600 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚251542566591106 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚249325382533120 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 125829120 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚41948160 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚64899171 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚262 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚249325365493760 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 G@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚65 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚21182550 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 D=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚60 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚253024919696896 PDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654602240 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚322 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚7717573 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚46132224 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 $ _timestamp‚1.786661109041361e+09 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 5368709120 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 2217201097346 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚30868172 >5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5264 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚4293ùæÿ§Ë&ÍÅM WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚253024919696896 NADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚59053056 H@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚665 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚28900188 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚429 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚322 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚34026706 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚41948160 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚221006277706240 IBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚60 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 2221831528960 B8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚262 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 2221831528960 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚250803029114880 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 5368709120 G@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚60 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚322 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚41348068 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚30868172 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 122905600 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚218656757874688 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17104896 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚253024860643840 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 4517085184 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚250803046219776 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚218656737230848 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 trainer/global_step‚9199 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚64899171 K>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59053056 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚34026704 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚251542624978534 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 2349519831552 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚262 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 D6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚322 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 5494538240 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚223 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚64894876 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚21182552 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚20643840 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521206 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚250803029114880 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 122905600 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚251542566591106 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚249325382533120 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 125829120 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚41948160 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚64899171 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚262 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚249325365493760 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 G@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚65 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚21182550 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 D=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚60 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚253024919696896 PDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654602240 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚322 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚7717573 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚46132224 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 $ _timestamp‚1.786661109041361e+09 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 5368709120 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 2217201097346 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚30868172 >5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5264 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚4293 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚209 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚34026704 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚250803046219776 EDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚28900123 SFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚66776064 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 831258624 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 2217242445414 _runtime‚2457.571691492 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚30872465 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚30872465 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 NGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚63 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 125829120 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 2221873477120 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚4293 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚221006210930176 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 2349473699328 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 5368709120 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚4295 EDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 2221873477120 B9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚5264 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 845178880 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 NADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚58387428 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 125829120 _step‚756 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17104896 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 5494538240ô1PC 2Î- _wandbruntime‚2457  _runtime‚2457XQÄqÏë /loss/FRt1/assignment_loss‚1.046492576071569 0loss/FRt2/assignment_loss‚0.9704182639639193 /loss/FRt2/detection_loss‚0.5013961366048212 %loss/total_loss‚3.066033457428463 trainer/global_step‚9199 _step‚757 /loss/FRt1/detection_loss‚0.5477264807881533 epoch‚22 % _timestamp‚1.7866611090423815e+09 _runtime‚2457.572208017"”êàvÐð /loss/FRt1/assignment_loss‚1.046492576071569 0loss/FRt2/assignment_loss‚0.9704182639639193 /loss/FRt2/detection_loss‚0.5013961366048212 %loss/total_loss‚3.066033457428463 trainer/global_step‚9199 _step‚757 /loss/FRt1/detection_loss‚0.5477264807881533 epoch‚22 % _timestamp‚1.7866611090423815e+09 _runtime‚2457.572208017õ!tRŸ•Ñ‚(:Š ÷‘ùÓÚë…á gpu.0.gpu‚47.0 gpu.0.memory‚5, gpu.0.memoryAllocated‚7.0313262939453125) gpu.0.memoryAllocatedBytes‚ 6039863296 gpu.0.temp‚33.0 gpu.0.powerWatts‚123.223' gpu.0.enforcedPowerLimitWatts‚500.0 gpu.0.powerPercent‚24.6446 gpu.0.smClock‚1410 gpu.0.memoryClock‚1593$ gpu.0.correctedMemoryErrors‚1955# gpu.0.uncorrectedMemoryErrors‚0Óˆ²âŸÒ‚(:” ÷‘ùÓõëä disk.sda2.out‚0 disk.sda3.in‚23471.75390625 disk.sda1.out‚0! disk.nvme0n1.in‚ 43465.2265625 disk.dm-1.out‚ 46.41015625# disk.dm-0.out‚55765.02880859375 disk.sda3.out‚ 46.41015625( disk.nvme0c0n1.out‚55765.02880859375 disk.sda1.in‚0 proc.cpu.threads‚17 network.recv‚ 4.708233e+06 disk.dm-1.in‚23471.75390625 disk.drbd2165.in‚0 disk.drbd2165.out‚0 disk.dm-0.in‚ 43465.2265625# disk.nvme0c0n1.in‚ 43465.2265625) proc.memory.availableMB‚ 5345.84765625# proc.memory.rssMB‚ 2286.19921875% disk./.usageGB‚487.22307205200195 disk.sda2.in‚0& disk.nvme0n1.out‚55765.02880859375 disk.sda.in‚23471.75390625 disk.sda.out‚ 46.41015625# memory_percent‚67.3715353012085* proc.memory.percent‚13.953852653503418 cpu‚27.13951786730511 network.sent‚ 1.2055817e+07) disk./.usagePercent‚32.69794917279665~cç2Ó- _wandbruntime‚2462  _runtime‚2462LÏâøàÔÚ /Purity/1FRt/event_purity‚0.3756634593009949 2Purity/2FRt/event_proportion‚0.040062315762043 particle/accuracy_1_of_1‚1 % _timestamp‚1.7866611143331382e+09 /Purity/*FRt/event_purity‚0.3716740608215332 .Purity/*FRt/FRt_purity‚0.38501104712486267 3Purity/0FRt/event_proportion‚0.3301943838596344 +jet/accuracy_1_of_1‚0.25403040647506714 _step‚758 -Purity/2FRt/FRt_purity‚0.4589686691761017 +jet/accuracy_2_of_2‚0.16632845997810364 trainer/global_step‚9199 +validation_accuracy‚0.16632845997810364 7validation_average_jet_accuracy‚0.25846990942955017 )loss/val_total_loss‚2.961112959384492 2Purity/*FRt/event_proportion‚0.669805645942688 -Purity/1FRt/FRt_purity‚0.3756634593009949 (particle/accuracy‚0.6070381999015808 3Purity/1FRt/event_proportion‚0.6297432780265808 /particle/accuracy_2_of_2‚0.6951428651809692 epoch‚22 _runtime‚2462.863266468 +jet/accuracy_1_of_2‚0.49229171872138977 Purity/0FRt/event_purity‚1 0Purity/2FRt/event_purity‚0.30955177545547485 /particle/accuracy_1_of_2‚0.9237805604934692 +particle/sensitivity‚0.6549814343452454 'particle/f_score‚0.5419220924377441 +particle/specificity‚0.5806440711021423:ú¡­Õß +validation_accuracy‚0.16632845997810364 7validation_average_jet_accuracy‚0.25846990942955017 )loss/val_total_loss‚2.961112959384492 2Purity/*FRt/event_proportion‚0.669805645942688 -Purity/1FRt/FRt_purity‚0.3756634593009949 (particle/accuracy‚0.6070381999015808 3Purity/1FRt/event_proportion‚0.6297432780265808 /particle/accuracy_2_of_2‚0.6951428651809692 epoch‚22 _runtime‚2462.863266468 +jet/accuracy_1_of_2‚0.49229171872138977 Purity/0FRt/event_purity‚1 0Purity/2FRt/event_purity‚0.30955177545547485 /particle/accuracy_1_of_2‚0.9237805604934692 +particle/sensitivity‚0.6549814343452454 'particle/f_score‚0.5419220924377441 +particle/specificity‚0.5806440711021423 /Purity/1FRt/event_purity‚0.3756634593009949 2Purity/2FRt/event_proportion‚0.040062315762043 particle/accuracy_1_of_1‚1 % _timestamp‚1.7866611143331382e+09 /Purity/*FRt/event_purity‚0.3716740608215332 .Purity/*FRt/FRt_purity‚0.38501104712486267 3Purity/0FRt/event_proportion‚0.330›ÄÈ1943838596344 +jet/accuracy_1_of_1‚0.25403040647506714 _step‚758 -Purity/2FRt/FRt_purity‚0.4589686691761017 +jet/accuracy_2_of_2‚0.16632845997810364 trainer/global_step‚9199ö§£' Ö‚(:• †’ùÓë°‚á gpu.0.gpu‚62.0 gpu.0.memory‚12, gpu.0.memoryAllocated‚7.0313262939453125) gpu.0.memoryAllocatedBytes‚ 6039863296 gpu.0.temp‚34.0 gpu.0.powerWatts‚144.36' gpu.0.enforcedPowerLimitWatts‚500.0) gpu.0.powerPercent‚28.872000000000003 gpu.0.smClock‚1410 gpu.0.memoryClock‚1593$ gpu.0.correctedMemoryErrors‚1955# gpu.0.uncorrectedMemoryErrors‚0È‹“ׂ(:ˆ †’ùÓØ„ãâ disk.sda.out‚ 46.80078125 disk.sda1.out‚0 disk.sda2.out‚0 disk.sda3.in‚23471.75390625# disk.nvme0c0n1.in‚ 43465.2265625 disk.sda2.in‚0# disk.nvme0n1.out‚55813.87890625$ memory_percent‚67.55199432373047# proc.memory.rssMB‚ 2315.30859375 disk.dm-0.in‚ 43465.2265625 disk.sda.in‚23471.75390625! disk.nvme0n1.in‚ 43465.2265625 disk.dm-1.out‚ 46.80078125* proc.memory.percent‚14.131522178649902 proc.cpu.threads‚17) disk./.usagePercent‚32.69793253230692 disk.dm-0.out‚55813.87890625 disk.sda3.out‚ 46.80078125% disk.nvme0c0n1.out‚55813.87890625 disk.sda1.in‚0 disk.dm-1.in‚23471.75390625& proc.memory.availableMB‚ 5316.28125 cpu‚27.14816882811744 network.sent‚ 1.2138509e+07 network.recv‚ 4.737298e+06$ disk./.usageGB‚487.2228240966797 disk.drbd2165.in‚0 disk.drbd2165.out‚0ûüõ2Ø- _wandbruntime‚2475  _runtime‚2475$Ä\ÇÙÁ & lr-AdamW/pg2‚0.0008638813085329851 trainer/global_step‚9249 % _timestamp‚1.7866611269681644e+09 _runtime‚2475.498176243 _step‚759 & lr-AdamW/pg1‚0.0008638813085329851‡ý:ÌÚÆ & lr-AdamW/pg1‚0.0008638813085329851 & lr-AdamW/pg2‚0.0008638813085329851 trainer/global_step‚9249 % _timestamp‚1.7866611269681644e+09 _runtime‚2475.498176243 _step‚759÷ae¿2Û- _wandbruntime‚2475  _runtime‚2475ØÂw¼'ܶO TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚41945088 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 831258624 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 2365596801536 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 122306526 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚31056461 F?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚60 @7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚5264 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚20643840 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚31056461 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 4654602240 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚222450925607424 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 2365645033984 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚262 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 _step‚760 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚31052174 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 2237254949888 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 4517085184 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚41945088 PCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚58387380 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚250989385908224 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚250989368868864 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚252483760881664 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 2237296894976 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚61478912 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 4628521206 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 2232639655463 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚34261407 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚4287 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚65317870 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚7766257 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚253221967176307 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 2232598307443 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚322 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚4287 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚322 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚223 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚220085349449728 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚220085328805888 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 4552982528 IBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚72 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚254721074881536 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚254721074881536 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚252483760881664 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 2237296894976 PCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚59049984 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚252483777986560 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4289 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚254721015831552 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚254721015831552 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚34261409 KBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚4876 G>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4876 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17104896 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚429 BDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚34261407 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚29097468 trainer/global_step‚9249 M@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚59049984 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚7766187 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 5368709120 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 4552982528 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚48232448 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 5368709120 KDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚60 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚31052174 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 125829120 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚322 D;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚5264 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 845178880 E8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚65313581 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚21331211 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 125829120 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚262 % _timestamp‚1.7866611269690933e+09 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚29097396 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 122905600 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚21331209 PIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚70 UHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚68876288 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 5494538240 IBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚60 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 4654602240 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚209 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 _runtime‚ 2475.50287791 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚252483777986560 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 2237254949888 JBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚665 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚222450994483712 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 IDeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4289 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚254721015831552 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚254721015831552 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚34261409 KBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚4876 G>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4876 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17104896 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚429 BDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚34261407 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚29097468 trainer/global_step‚9249 M@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚59049984 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚7766187 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 5368709120 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 4552982528 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚48232448 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 5368709120 KDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚60 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚31052174 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 125829120 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚322 D;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚5264 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 845178880 E8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚65313581 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚21331211 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 125829120 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚262 % _timestamp‚1.7866611269690933e+09 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚29097396 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 122905600 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚21331209 PIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚70 UHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚68876288 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 5494538240 IBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚60 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 4654602240 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚209 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 _runtime‚ 2475.50287791 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚252483777986560 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 2237254949888 JBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚665 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚222450994483712 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 IDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521206 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 125829120 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚254747628291072 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚65320421 G@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚57 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚251015571701760 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚41348068 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 5494538240 K>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59053056 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚31059754 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚429 SFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚64678912 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 2365846346240 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654602240 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚21333416 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚65324716 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚34264960 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚262 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚4293 >5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5264 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚31059754 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚222474287203840 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 5368709120 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 122905600 EDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚29100410 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚222474222524928 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚322 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚322 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚252510100324352 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 125829120 trainer/global_step‚9249 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚252510100324352 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 4552982528 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 125829120 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚262 _runtime‚2475.665975603 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 122905600 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚31055461 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 4654602240 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚41948160 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚29100467 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚7766996 % _timestamp‚1.7866611271355894e+09 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚34264960 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚65324716 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚34264962 H@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚665 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚7767051 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 _step‚761 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 2237527966720 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚65320421 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17104896 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚252510083219456 B9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚5264 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚322 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 4517085184 DDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚254747628291072 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚44035072 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚253248441929883 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚252510083219456 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚220108376178688 G@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚60 D=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚60 EDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚31055461ñ ÝCDàÆM SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 122905600 EDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚29100410 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚222474222524928 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚322 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚322 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚252510100324352 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 125829120 trainer/global_step‚9249 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚252510100324352 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 4552982528 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 125829120 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚262 _runtime‚2475.665975603 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 122905600 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚31055461 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 4654602240 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚41948160 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚29100467 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚7766996 % _timestamp‚1.7866611271355894e+09 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚34264960 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚65324716 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚34264962 H@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚665 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚7767051 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 _step‚761 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 2237527966720 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚65320421 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17104896 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚252510083219456 B9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚5264 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚322 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 4517085184 DDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚254747628291072 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚44035072 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚253248441929883 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚252510083219456 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚220108376178688 G@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚60 D=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚60 EDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚31055461 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚21333414 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚34264962 PDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521206 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 125829120 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚254747628291072 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚65320421 G@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚57 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚251015571701760 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚41348068 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 5494538240 K>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59053056 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚31059754 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚429 SFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚64678912 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 2365846346240 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654602240 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚21333416 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚65324716 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚34264960 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚262 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚4293 >5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5264 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚31059754 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚222474287203840 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 5368709120ù06ò©2á- _wandbruntime‚2475  _runtime‚2475îͳwrâì /loss/FRt2/detection_loss‚0.5098131798742821 epoch‚23 trainer/global_step‚9249 $ _timestamp‚1.786661127136707e+09 _step‚762 0loss/FRt1/assignment_loss‚0.8431372012733171 0loss/FRt2/assignment_loss‚0.8358490539475687 /loss/FRt1/detection_loss‚0.5710915536420468 &loss/total_loss‚2.7598909887372147 _runtime‚2475.666511564 ôB_wãñ /loss/FRt1/detection_loss‚0.5710915536420468 &loss/total_loss‚2.7598909887372147 _runtime‚2475.666511564 /loss/FRt2/detection_loss‚0.5098131798742821 epoch‚23 trainer/global_step‚9249 $ _timestamp‚1.786661127136707e+09 _step‚762 0loss/FRt1/assignment_loss‚0.8431372012733171 0loss/FRt2/assignment_loss‚0.8358490539475687úí+™2ä- _wandbruntime‚2487  _runtime‚2487Æy?|Åå¿ _step‚763 % lr-AdamW/pg1‚0.000857936709869976 % lr-AdamW/pg2‚0.000857936709869976 trainer/global_step‚9299 % _timestamp‚1.7866611394175515e+09 _runtime‚2487.949898941tÂCÊæÄ _runtime‚2487.949898941 _step‚763 % lr-AdamW/pg1‚0.000857936709869976 % lr-AdamW/pg2‚0.000857936709869976 trainer/global_step‚9299 % _timestamp‚1.7866611394175515e+09û™\Ó2ç- _wandbruntime‚2487  _runtime‚2487€.½'è·O G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4289 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 5368709120 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚7806961 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚254542877051095 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚322 % _timestamp‚1.7866611394182205e+09 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚254542818663715 C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚4289 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚31221563 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 5494538240 @7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚5264 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚253799877771264 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 2377917465088 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 2377873426944 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚34439059 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚29248487 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 125829120 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚262 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 2248841715712 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚21441461 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚256048778536960 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚221237718089728 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 4517085184 IDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4876 M@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚59049984 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚253799894876160 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚34439059 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 831258624 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17104896 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚223615570872832 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚44038144 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 F>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚41945088 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚253799894876160 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17104896 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 4552982528 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 4628521206 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚256048719486976 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚256048719486976 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 4654602240 JBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚665 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚252298658545664 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚322 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚252298675585024 _step‚764 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 2248841715712 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚209 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚221237697445888 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 2244160118051 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚34439057 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 2244201466071 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚256048778536960 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚31217276 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 4552982528 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚31217276 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚61478912 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 125829120 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚322 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 5494538240 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 5494538240 _runtime‚2487.949991206 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚429 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚4287 PIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚65 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 122306526 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 125829120 PCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚59049984 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚20643840 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚41348020 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 5368709120 KBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚4876 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚223 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚7807026 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 F?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚60 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 2248883660800 UHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚64681984 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 5368709120 IBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚60 trainer/global_step‚9299 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚253799877771264 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 1229056002鬶Â'é¼O @7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚5264 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚253799877771264 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 2377917465088 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 2377873426944 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚34439059 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚29248487 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 125829120 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚262 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 2248841715712 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚21441461 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚256048778536960 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚221237718089728 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 4517085184 IDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4876 M@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚59049984 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚253799894876160 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚34439059 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 831258624 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17104896 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚223615570872832 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚44038144 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 F>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚41945088 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚253799894876160 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17104896 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 4552982528 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 4628521206 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚256048719486976 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚256048719486976 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 4654602240 JBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚665 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚252298658545664 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚322 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚252298675585024 _step‚764 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 2248841715712 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚209 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚221237697445888 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 2244160118051 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚34439057 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 2244201466071 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚256048778536960 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚31217276 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 4552982528 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚31217276 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚61478912 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 125829120 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚322 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 5494538240 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 5494538240 _runtime‚2487.949991206 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚429 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚4287 PIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚65 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 122306526 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 125829120 PCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚59049984 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚20643840 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚41348020 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 5368709120 KBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚4876 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚223 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚7807026 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 F?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚60 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 2248883660800 UHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚64681984 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 5368709120 IBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚60 trainer/global_step‚9299 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚253799877771264 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 122905600 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4289 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 5368709120 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚7806961 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚254542877051095 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚322 % _timestamp‚1.7866611394182205e+09 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚254542818663715 C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚4289 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚31221563 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 5494538240ü«ÅáA2ê- _wandbruntime‚2488  _runtime‚24887;+eëÁM NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚34442612 D6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚322 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 2249114732544 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚209 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚4295 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚31220563 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚41348068 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚34442610 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚253826217213952 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚65663173 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚223638868519424 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 831258624 SEDeviceStatsMonitor.on_trÒå`¦b!ain_batch_end/requested_bytes.small_pool.peak‚ 122306526 SFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚56290304 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 2378159347200 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚65663173 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚252324861378560 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 2244432038731 _runtime‚2488.115115291 K>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59053056 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 4552982528 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521206 IBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚60 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚31224856 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚4293 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚31220563 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚253826200109056 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚221260765462528 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚41948160 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚254569293417291 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 trainer/global_step‚9299 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17104896 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 2249114732544 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚20643840 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 2378123700736 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚252324844339200 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 G@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚60 D=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚60 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 4552982528 G@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚60 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚21443666 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚223 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 NADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚58387428 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 5494538240 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17104896 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚29251519 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚262 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654602240 EDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚253826217213952 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚256075331946496 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 122905600 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚256075272893440 B5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5264 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 4654602240 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚41948160 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚34442610 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚29251459 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 2244390690663 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚253826200109056 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚21443664 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚34442612 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚262 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚65667468 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚322 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚256075331946496 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 2249072784384 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 122905600 B9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚5264 NGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚58 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 5368709120 _step‚765 I@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚4876 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚254569235029863 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 5494538240÷ÝVÌ&ìÆM G@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚60 D=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚60 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 4552982528 G@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚60 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚21443666 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚223 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 NADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚58387428 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 5494538240 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17104896 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚29251519 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚262 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654602240 EDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚253826217213952 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚256075331946496 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 122905600 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚256075272893440 B5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5264 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 4654602240 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚41948160 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚34442610 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚29251459 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 2244390690663 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚253826200109056 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚21443664 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚34442612 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚262 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚65667468 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚322 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚256075331946496 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 2249072784384 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 122905600 B9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚5264 NGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚58 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 5368709120 _step‚765 I@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚4876 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚254569235029863 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 5494538240 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚34442612 D6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚322 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 2249114732544 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚209 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚4295 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚31220563 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚41348068 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚34442610 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚253826217213952 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚65663173 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚223638868519424 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 831258624 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 122306526 SFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚56290304 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 2378159347200 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚65663173 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚252324861378560 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 2244432038731 _runtime‚2488.115115291 K>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59053056 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 4552982528 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521206 IBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚60 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚31224856 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚4293 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚31220563 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚253826200109056 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚221260765462528 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚41948160 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚254569293417291 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 trainer/global_step‚9299 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17104896 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 2249114732544 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚20643840 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 2378123700736 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚252324844339200 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0ýP‹[2í- _wandbruntime‚2488  _runtime‚2488å®^Trîì /loss/FRt1/detection_loss‚0.5191102334776685 &loss/total_loss‚2.6068126830078366 epoch‚23 trainer/global_step‚9299 % _timestamp‚1.7866611395857518e+09 _runtime‚ 2488.11558041 _step‚766 0loss/FRt1/assignment_loss‚0.7846597302513779 0loss/FRt2/assignment_loss‚0.7857740049636098 /loss/FRt2/detection_loss‚0.51726871431518059õŸ«wïñ _step‚766 0loss/FRt1/assignment_loss‚0.7846597302513779 0loss/FRt2/assignment_loss‚0.7857740049636098 /loss/FRt2/detection_loss‚0.5172687143151805 /loss/FRt1/detection_loss‚0.5191102334776685 &loss/total_loss‚2.6068126830078366 epoch‚23 trainer/global_step‚9299 % _timestamp‚1.7866611395857518e+09 _runtime‚ 2488.11558041þ%êøæ•ð‚(:Š •’ùÓ‰™œá gpu.0.gpu‚15.0 gpu.0.memory‚3, gpu.0.memoryAllocated‚7.0313262939453125) gpu.0.memoryAllocatedBytes‚ 6039863296 gpu.0.temp‚34.0 gpu.0.powerWatts‚222.799' gpu.0.enforcedPowerLimitWatts‚500.0 gpu.0.powerPercent‚44.5598 gpu.0.smClock‚1410 gpu.0.memoryClock‚1593$ gpu.0.correctedMemoryErrors‚1955# gpu.0.uncorrectedMemoryErrors‚0…ê:ê”ñ‚(:‰ •’ùÓ’›¦ã disk.sda2.in‚0! disk.nvme0n1.in‚ 43465.2265625# disk.nvme0n1.out‚55817.45703125 disk.dm-1.in‚23471.75390625 network.sent‚ 1.218771e+07$ disk./.usageGB‚487.2857093811035 disk.dm-0.out‚55817.45703125 disk.sda3.out‚ 47.01953125 disk.sda.out‚ 47.01953125 disk.drbd2165.out‚0$ memory_percent‚67.55242347717285( proc.memory.availableMB‚ 5316.2109375% disk.nvme0c0n1.out‚55817.45703125 disk.sda1.in‚0 disk.sda1.out‚0 disk.sda2.out‚0 disk.dm-1.out‚ 47.01953125 disk.drbd2165.in‚0* proc.memory.percent‚14.131522178649902 disk.sda3.in‚23471.75390625# disk.nvme0c0n1.in‚ 43465.2265625# proc.memory.rssMB‚ 2315.30859375 cpu‚27.14777227884789 proc.cpu.threads‚17 network.recv‚ 4.762885e+06) disk./.usagePercent‚32.70215281650887 disk.dm-0.in‚ 43465.2265625 disk.sda.in‚23471.75390625Á§x2ò- _wandbruntime‚2500  _runtime‚2500,ïìÆóÀ _step‚767 & lr-AdamW/pg1‚0.0008519851786337881 & lr-AdamW/pg2‚0.0008519851786337881 trainer/global_step‚9349 $ _timestamp‚1.786661151990086e+09 _runtime‚2500.520124944µÜZ±ËôÅ & lr-AdamW/pg1‚0.0008519851786337881 & lr-AdamW/pg2‚0.0008519851786337881 trainer/global_step‚9349 $ _timestamp‚1.786661151990086e+09 _runtime‚2500.520124944 _step‚767ÿ:f 2õ- _wandbruntime‚2500  _runtime‚2500Ùz•¼'ö¶O TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 125829120 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 2255763276679 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 2260428481536 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚29399399 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 5368709120 F?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚60 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚223 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 122905600 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 IBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚60 E8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚65999085 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚255115994660864 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 4654602240 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚253607965261824 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 5494538240 @7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚5264 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 5368709120 IDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 4654602240 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 4552982528 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚222390086729728 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚20643840 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 2390212486144 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 5494538240 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚322 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚255115994660864 KBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚4876 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚21551711 PIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚57 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 5494538240 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚262 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚257376423142400 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 2260470426624 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 % _timestamp‚1.7866611519910126e+09 D;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚5264 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 122306526 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17104896 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚253607948222464 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 125829120 KDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚60 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚34616709 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚44038144 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚61478912 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 _runtime‚ 2500.52487975 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚34616707 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚257376482192384 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚34616709 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚31382378 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚21551709 BDeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4289 IDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4876 PCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚59049984 _step‚768 IBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚59 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚222390066085888 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚255863728538503 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 2255721928659 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 5368709120 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚257376482192384 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 122905600 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 PCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚58387380 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚322 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 2260470426624 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚4287 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚224780299215872 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 831258624 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 2390168448000 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚262 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 2260428481536 trainer/global_step‚9349 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚4287 UHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚64681984 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚255863670151123 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚31382378 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚255116011765760 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚29399340 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚31386665 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 845178880 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚4289 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚41945088 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚66003374 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚7847631Ò¸9‚÷»O [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 2260470426624 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 % _timestamp‚1.7866611519910126e+09 D;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚5264 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 122306526 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17104896 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚253607948222464 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 125829120 KDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚60 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚34616709 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚44038144 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚61478912 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 _runtime‚ 2500.52487975 L?DeviceStatsMonitor.on_train_batch_start/a·7n?"ctive.large_pool.freed‚34616707 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚257376482192384 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚34616709 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚31382378 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚21551709 BDeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4289 IDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4876 PCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚59049984 _step‚768 IBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚59 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚222390066085888 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚255863728538503 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 2255721928659 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 5368709120 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚257376482192384 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 122905600 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 PCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚58387380 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚322 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 2260470426624 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚4287 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚224780299215872 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 831258624 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 2390168448000 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚262 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 2260428481536 trainer/global_step‚9349 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚4287 UHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚64681984 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚255863670151123 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚31382378 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚255116011765760 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚29399340 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚31386665 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 845178880 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚4289 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚41945088 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚66003374 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚7847631 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 125829120 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 2255763276679 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 2260428481536 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚29399399 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 5368709120 F?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚60 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚223 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 122905600 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 IBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚60 E8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚65999085 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚255115994660864 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 4654602240 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚253607965261824 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 5494538240 @7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚5264 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 5368709120 IDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 4654602240 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 4552982528 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚222390086729728 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚20643840 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 2390212486144 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 5494538240 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚322 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚255115994660864 KBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚4876 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚21551711 PIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚57 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 5494538240 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚262 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚257376423142400€¸’^t2ø- _wandbruntime‚2500  _runtime‚2500ù)£Æ&ùÀM [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚255142334103552 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚257403035601920 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚29402473 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 125829120 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 2260659550208 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚209 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚224803591780352 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 2390409448448 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚322 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚31385665 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 2260659550208 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚34620262 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654602240 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 5368709120 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚4295 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚255890144904699 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚31389958 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 4552982528 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚262 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚66005925 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚66010220 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚322 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17104896 EDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59053056 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚257402976548864 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚48229376 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚253634151055360 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 5494538240 D=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚60 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚4293 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 122905600 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚222413134102528 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 NADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚58387428 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 5494538240 IBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚60 $ _timestamp‚1.786661152153824e+09 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚31385665 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 845178880 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚34620260 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 122905600 NGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚76 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚7848481 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 BDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚29402395 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚257403035601920 DDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 125829120 >5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5264 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 SFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚68873216 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚255890086517271 trainer/global_step‚9349 _runtime‚2500.684153927 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚20643840 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚222413113458688 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚262 EDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 H@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚665 G@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚78 B9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚5264 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 122306526 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚255142334103552 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚255142316998656 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚66005925 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚429 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 PDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521206 G@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚60 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚34620260 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 2390457677824 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚4293 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚224803522907136 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚322N§Ë&úÅM J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚34620260 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 122905600 NGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚76 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚7848481 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 BDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚29402395 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚257403035601920 DDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 125829120 >5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5264 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 SFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚68873216 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚255890086517271 trainer/global_step‚9349 _runtime‚2500.684153927 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚20643840 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚222413113458688 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚262 EDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 H@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚665 G@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚78 B9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚5264 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 122306526 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚255142334103552 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚255142316998656 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚66005925 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚429 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 PDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521206 G@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚60 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚34620260 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 2390457677824 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚4293 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚224803522907136 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚322 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚255142334103552 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚257403035601920 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚29402473 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 125829120 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 2260659550208 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚209 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚224803591780352 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 2390409448448 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚322 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚31385665 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 2260659550208 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚34620262 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654602240 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 5368709120 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚4295 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚255890144904699 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚31389958 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 4552982528 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚262 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚66005925 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚66010220 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚322 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17104896 EDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59053056 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚257402976548864 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚48229376 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚253634151055360 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 5494538240 D=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚60 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚4293 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 122905600 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚222413134102528 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 NADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚58387428 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 5494538240 IBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚60 $ _timestamp‚1.786661152153824e+09 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚31385665 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 845178880ÌåÇ>2û- _wandbruntime‚2500  _runtime‚2500&››qüë 0loss/FRt1/assignment_loss‚0.7417499120619632 0loss/FRt2/assignment_loss‚0.6940475254671232 %loss/total_loss‚2.521837891306488 trainer/global_step‚9349 $ _timestamp‚1.786661152154847e+09 _runtime‚2500.684814253 _step‚770 /loss/FRt1/detection_loss‚0.5588650100359999 /loss/FRt2/detection_loss‚0.5271754437414018 epoch‚23)'âvýð /loss/FRt1/detection_loss‚0.5588650100359999 /loss/FRt2/detection_loss‚0.5271754437414018 epoch‚23 0loss/FRt1/assignment_loss‚0.7417499120619632 0loss/FRt2/assignment_loss‚0.6940475254671232 %loss/total_loss‚2.521837891306488 trainer/global_step‚9349 $ _timestamp‚1.786661152154847e+09 _runtime‚2500.684814253 _step‚770‚Ý&cÄ¡þ‚(:– ¤’ùÓ‚¥üà gpu.0.gpu‚61.0 gpu.0.memory‚11, gpu.0.memoryAllocated‚7.0313262939453125) gpu.0.memoryAllocatedBytes‚ 6039863296 gpu.0.temp‚32.0 gpu.0.powerWatts‚119.166' gpu.0.enforcedPowerLimitWatts‚500.0) gpu.0.powerPercent‚23.833199999999998 gpu.0.smClock‚1410 gpu.0.memoryClock‚1593$ gpu.0.correctedMemoryErrors‚1955# gpu.0.uncorrectedMemoryErrors‚0Š€¯˜ÿ‚(: ¤’ùÓ‰•Òâ" disk.nvme0n1.in‚43477.98046875 proc.cpu.threads‚17 disk.dm-0.in‚43477.98046875' disk.nvme0c0n1.out‚55863.4404296875 disk.sda1.out‚0 disk.dm-1.in‚ 23492.6171875 disk.drbd2165.out‚0$ memory_percent‚67.55256652832031# proc.memory.rssMB‚ 2315.30859375* proc.memory.percent‚14.131522178649902 disk.sda2.in‚0 disk.sda2.out‚0% disk.nvme0n1.out‚55863.4404296875 disk.dm-1.out‚ 47.46484375 disk.drbd2165.in‚0 network.sent‚ 1.2283353e+07 network.recv‚ 4.795806e+06 disk.sda3.in‚ 23492.6171875 disk.sda.out‚ 47.46484375 disk.sda1.in‚0% proc.memory.availableMB‚ 5316.1875 cpu‚27.14779999605433) disk./.usagePercent‚32.70199946799586$ disk./.usageGB‚487.2834243774414" disk.dm-0.out‚55863.4404296875 disk.sda3.out‚ 47.46484375$ disk.nvme0c0n1.in‚43477.98046875 disk.sda.in‚ 23492.6171875ô¯Ô72€- _wandbruntime‚2512  _runtime‚2512Oì4)ÇÁ _runtime‚2512.966533923 _step‚771 & lr-AdamW/pg1‚0.0008460270970801513 & lr-AdamW/pg2‚0.0008460270970801513 trainer/global_step‚9399 % _timestamp‚1.7866611644364958e+092"ôÌ‚Æ & lr-AdamW/pg1‚0.0008460270970801513 & lr-AdamW/pg2‚0.0008460270970801513 trainer/global_step‚9399 % _timestamp‚1.7866611644364958e+09 _runtime‚2512.966533923 _step‚771ƒ€ØM}2ƒ- _wandbruntime‚2512  _runtime‚25129iVóS„·O PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚31551767 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4289 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 2402482529792 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 125829120 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 2272015247360 D;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚5264 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 4517085184 _runtime‚2512.970892732 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 G>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4876 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 4552982528 KDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚60 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚223542434725888 JBDeviceStatsMonitor.on_train_batch_start/segment.large_póyùj"ool.current‚262 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 122905600 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚66346126 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚4287 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚429 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 845178880 UHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚68876288 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚254917237899264 PCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚59049984 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚34794359 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚41945088 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 122905600 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚34794357 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚223 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚7888548 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 2272057192448 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17104896 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚256432111550464 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚31547480 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚29550438 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚254917254938624 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 125829120 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 M@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚59049984 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚258704185847808 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 831258624 C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚4289 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚21661959 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚41348020 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚322 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚20643840 BDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚258704126797824 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚209 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚34794357 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚48232448 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 125829120 trainer/global_step‚9399 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚256432128655360 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚256432111550464 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 5494538240 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚262 IDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚257184580025911 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 5494538240 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 IBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚60 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚31547480 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17104896 PIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚69 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 2402434297344 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚61478912 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 5368709120 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 5368709120 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚322 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚258704126797824 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 2272057192448 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚21661961 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚225944869023232 _step‚772 IDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚256432128655360 IBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚71 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 4628521206 PCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚58387380 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚322 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚257184521638531 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2ú5²Â'…¼O C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚4289 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚21661959 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚41348020 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚322 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚20643840 BDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚258704126797824 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚209 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚34794357 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚48232448 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 125829120 trainer/global_step‚9399 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚256432128655360 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚256432111550464 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 5494538240 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚262 IDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚257184580025911 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 5494538240 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 IBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚60 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚31547480 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17104896 PIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚69 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 2402434297344 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚61478912 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 5368709120 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 5368709120 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚322 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚258704126797824 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 2272057192448 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚21661961 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚225944869023232 _step‚772 IDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚256432128655360 IBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚71 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 4628521206 PCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚58387380 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚322 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚257184521638531 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚31551767 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4289 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 2402482529792 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 125829120 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 2272015247360 D;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚5264 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 4517085184 _runtime‚2512.970892732 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 G>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4876 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 4552982528 KDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚60 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚223542434725888 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚262 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 122905600 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚66346126 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚4287 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚429 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 845178880 UHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚68876288 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚254917237899264 PCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚59049984 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚34794359 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚41945088 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 122905600 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚34794357 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚223 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚7888548 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 2272057192448 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17104896 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚256432111550464 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚31547480 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚29550438 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚254917254938624 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 125829120 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 M@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚59049984 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚258704185847808 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 831258624„ÌJ†°2†- _wandbruntime‚2513  _runtime‚2513%‘K‚Ç&‡ÁM G@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚69 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 2402723691008 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521206 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚322 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚34797912 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚21664166 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚254943423692800 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚41348068 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 % _timestamp‚1.7866611645999959e+09 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚429 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 DDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚29553453 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚223565482098688 NADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚58387428 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚262 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚66348677 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚41948160 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 845178880 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚262 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚29553522 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚256458433888256 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚34797912 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚31550767 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚257210996392107 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚322 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚322 D=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚60 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚20643840 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 2267514311879 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 125829120 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17104896 H@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚665 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚21664164 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚7889289 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚61478912 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 5494538240 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 125829120 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚4293 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 4517085184 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 5368709120 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚262 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 K>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59053056 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚31555060 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 831258624 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚257210938004679 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 5494538240 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 122905600 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚258730739257344 B9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚5264 I@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚4876 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚66352972 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚256458450993152 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚256458433888256 >5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5264 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654602240 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17104896 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 122306526 IBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚60 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚41948160 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚225968226433536 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚225968163851776 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 5494538240 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚256458450993152 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚223 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚322 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 125829120 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 4654602240 NADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚59053056 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 trainer/global_step‚9399 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚4295 E8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 BDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 G@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚60 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚66352972 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚7889356 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚41937920 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚254943440732160 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚31550767 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 4552982528 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚4293 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 SFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚62581760 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 5368709120 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 2272246316032 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0°ù%¸ˆÆM J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚31550767 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 4552982528 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚4293 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 SFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚62581760 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 5368709120 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 2272246316032 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 G@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚69 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 2402723691008 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521206 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚322 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚34797912 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚21664166 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚254943423692800 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚41348068 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 % _timestamp‚1.7866611645999959e+09 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚429 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 DDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚29553453 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚223565482098688 NADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚58387428 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚262 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚66348677 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚41948160 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 845178880 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 d½1?JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚262 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚29553522 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚256458433888256 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚34797912 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚31550767 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚257210996392107 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚322 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚322 D=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚60 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚20643840 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 2267514311879 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 125829120 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17104896 H@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚665 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚21664164 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚7889289 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚61478912 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 5494538240 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 125829120 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚4293 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 4517085184 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 5368709120 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚262 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 K>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59053056 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚31555060 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 831258624 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚257210938004679 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 5494538240 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 122905600 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚258730739257344 B9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚5264 I@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚4876 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚66352972 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚256458450993152 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚256458433888256 >5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5264 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654602240 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17104896 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 122306526 IBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚60 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚41948160 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚225968226433536 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚225968163851776 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 5494538240 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚256458450993152 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚223 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚322 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 125829120 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 4654602240 NADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚59053056 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 trainer/global_step‚9399 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚4295 E8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 BDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 G@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚60 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚66352972 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚7889356 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚41937920 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚254943440732160…)æ 2‰- _wandbruntime‚2513  _runtime‚2513Ú3kÞrŠì _runtime‚2513.130982553 _step‚774 &loss/total_loss‚2.7227224245083943 epoch‚23 % _timestamp‚1.7866611646010807e+09 0loss/FRt1/assignment_loss‚0.8308552509950908 0loss/FRt2/assignment_loss‚0.7737149616406104 .loss/FRt1/detection_loss‚0.557009766671775 /loss/FRt2/detection_loss‚0.5611424452009179 trainer/global_step‚9399·£-w‹ñ _step‚774 &loss/total_loss‚2.7227224245083943 epoch‚23 % _timestamp‚1.7866611646010807e+09 0loss/FRt1/assignment_loss‚0.8308552509950908 0loss/FRt2/assignment_loss‚0.7737149616406104 .loss/FRt1/detection_loss‚0.557009766671775 /loss/FRt2/detection_loss‚0.5611424452009179 trainer/global_step‚9399 _runtime‚2513.130982553†+¿h Œ‚(:• ³’ùÓû¥Œá gpu.0.gpu‚37.0 gpu.0.memory‚6, gpu.0.memoryAllocated‚7.0313262939453125) gpu.0.memoryAllocatedBytes‚ 6039863296 gpu.0.temp‚34.0 gpu.0.powerWatts‚218.741' gpu.0.enforcedPowerLimitWatts‚500.0) gpu.0.powerPercent‚43.748200000000004 gpu.0.smClock‚1410 gpu.0.memoryClock‚1593$ gpu.0.correctedMemoryErrors‚1955# gpu.0.uncorrectedMemoryErrors‚0ª(–‚(:‹ ³’ùÓûôŸä cpu‚27.147484283022212 disk.sda.in‚ 24398.3515625$ disk.nvme0n1.out‚55867.240234375 disk.drbd2165.in‚0! disk.dm-0.out‚55867.240234375 disk.sda3.in‚ 24398.3515625 disk.sda3.out‚ 47.609375$ memory_percent‚67.55259037017822# proc.memory.rssMB‚ 2315.30859375* disk./.usagePercent‚32.702039405171206$ disk.nvme0c0n1.in‚43497.19921875& disk.nvme0c0n1.out‚55867.240234375 disk.sda.out‚ 47.609375 disk.sda1.in‚0 disk.sda2.in‚0) proc.memory.availableMB‚ 5316.18359375* proc.memory.percent‚14.131522178649902 disk.sda1.out‚0 disk.dm-1.in‚ 24398.3515625 disk.dm-0.in‚43497.19921875 proc.cpu.threads‚17 network.sent‚ 1.2347988e+07 network.recv‚ 4.817321e+06% disk./.usageGB‚487.28401947021484 disk.sda2.out‚0" disk.nvme0n1.in‚43497.19921875 disk.dm-1.out‚ 47.609375 disk.drbd2165.out‚0Î,2Ž- _wandbruntime‚2525  _runtime‚2525<ß1ÆÀ trainer/global_step‚9449 $ _timestamp‚1.786661177352608e+09 _runtime‚2525.882686191 _step‚775 & lr-AdamW/pg1‚0.0008400628478855105 & lr-AdamW/pg2‚0.00084006284788551055þG/ËÅ _step‚775 & lr-AdamW/pg1‚0.0008400628478855105 & lr-AdamW/pg2‚0.0008400628478855105 trainer/global_step‚9449 $ _timestamp‚1.786661177352608e+09 _runtime‚2525.882686191‡Š¾ 2‘- _wandbruntime‚2525  _runtime‚2525¡«ª½'’·O `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚257748245544960 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚29701331 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚224694824009728 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚257748228440064 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚21772211 PIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚66 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 5494538240 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚322 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 122905600 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚66688878 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 4628521206 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 122306526 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 KDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚60 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚260031830453248 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚257748245544960 BDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 G>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4876 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 2283643958272 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 2283602013184 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 4654602240 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17104896 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚34972007 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚262 % _timestamp‚1.7866611773536637e+09 IDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚260031830453248 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚257748228440064 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 4552982528 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4289 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚258505431513319 _runtime‚2525.886903212 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚34972009 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚4287 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚7929120 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚227109566814720 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 5494538240 F?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚60 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚46135296 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 2278886897895 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 5368709120 IBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚60 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 122905600 UHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚66779136 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚262 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 PCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚58387380 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚41348020 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚262 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚4287 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 2283602013184 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚21772209 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 2414696669696 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚322 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚256226544615424 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚31712582 PCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚59049984 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚29701263 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚429 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 4654602240 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚41945088 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚31712582 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 D;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚5264 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 125829120 C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚4289 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚31716869 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 5368709120 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 4552982528 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚223 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 845178880 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚61478912 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 4517085184 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚322 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚34972009 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚260031889503232 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚260031889503232 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚20643840 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 125829120 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 IBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚68 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚322 IDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚260031830453248 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚257748228440064 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 4552982528 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4289 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚258505431513319 _runtime‚2525.886903212 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚34972009 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚4287 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚7929120 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚227109566814720 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 5494538240 F?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚60 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚46135296 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 2278886897895 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 5368709120 IBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚60 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 122905600 UHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚66779136 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚262 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 PCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚58387380 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚41348020 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚262 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚4287 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 2283602013184 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚21772209 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 2414696669696 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚322 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚256226544615424 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚31712582 PCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚59049984 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚29701263 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚429 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 4654602240 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚41945088 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚31712582 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 D;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚5264 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 125829120 C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚4289 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚31716869 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 5368709120 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 4552982528 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚223 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 845178880 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚61478912 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 4517085184 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚322 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚34972009 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚260031889503232 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚260031889503232 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚20643840 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 125829120 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 IBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚68 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚322 IDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 G>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4876 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 2283643958272 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 2283602013184 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 4654602240ˆf;+æ2”- _wandbruntime‚2526  _runtime‚2526ç ë“ •ÁM RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 125829120 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚34975562 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 2283833081856 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 5494538240 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 trainer/global_step‚9449 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚29704254 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 2283875030016 B9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚5264 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚209 _runtime‚2526.064880788 B6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚322 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚31715869 EDevà^È 4iceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654602240 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚227132788408832 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚224717871382528 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 5494538240 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 E5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5264 SFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚70970368 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 % _timestamp‚1.7866611775344489e+09 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚41948160 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚66695724 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚256252713369600 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚322 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚260058442912768 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚322 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚322 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚262 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚31715869 I@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚4876 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 2283833081856 NGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚68 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚61478912 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚257774567882752 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚260058383859712 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚21774416 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521206 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚262 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 4552982528 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚256252730408960 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 122306526 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 5368709120 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 125829120 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚4295 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚34975560 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚257774550777856 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 K>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59053056 P8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚429 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚7929840 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 5368709120 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17104896 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 2279117470555÷BëÌ&–ÆM H@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚665 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚21774414 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚7929908 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚258531789492087 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚227132859379200 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 831258624 NADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚58387428 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚258531847879515 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚66695724 D5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5264 SFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚70970368 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 % _timestamp‚1.7866611775344489e+09 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚41948160 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚66695724 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚256252713369600 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚322 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚260058442912768 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚322 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚322 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚262 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚31715869 I@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚4876 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 2283833081856 NGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚68 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚61478912 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚257774567882752 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚260058383859712 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚21774416 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521206 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚262 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 4552982528 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚256252730408960 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 122306526 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 5368709120 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 125829120 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚4295 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚34975560 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚257774550777856 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 K>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59053056 P8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚429 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚7929840 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 5368709120 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17104896 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 2279117470555 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 125829120 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚34975562 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 2283833081856 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 5494538240 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 trainer/global_step‚9449 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚29704254 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 2283875030016 B9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚5264 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚209 _runtime‚2526.064880788 B6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚322 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚31715869 EDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654602240 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚227132788408832 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚224717871382528 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 5494538240 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 EDeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4289 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚262 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚223 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚228274235786240 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 2290407360483 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚262 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚46135296 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 4517085184 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 2290448708503 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 5368709120 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 125829120 $ _timestamp‚1.786661189859186e+09 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚35149659 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚259064345329664 PCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚59049984 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚7969630 IBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚60 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 4654602240 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 2295230724096 UHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚66779136 BDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17104896 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚4287 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚21882459 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 5494538240 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚322 M@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚59049984 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚261359534108672 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 4552982528 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 4654602240 KBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚4876 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚257535834292224 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚257535817252864 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 4552982528 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚41945088 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚209 @7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚5264 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 JBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚665 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚429 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚7969569 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 5368709120 G>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4876 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 2295188779008 IDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17104896 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚4287 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚21882459 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 5494538240 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚322 M@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚59049984 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚261359534108672 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 4552982528 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 4654602240 KBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚4876 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚257535834292224 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚257535817252864 VGDeviceStatsMonitor.on_train_b§sý[atch_start/allocated_bytes.large_pool.peak‚ 4552982528 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚41945088 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚209 @7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚5264 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 JBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚665 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚429 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚7969569 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 5368709120 G>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4876 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 2295188779008 IDeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4289 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚262 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚223 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚228274235786240 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 2290407360483 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚262Œ÷c)y2¢- _wandbruntime‚2538  _runtime‚2538&eÁ5Ç&£ÁM E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 122306526 trainer/global_step‚9499 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚21884666 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚21884664 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 125829120 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚322 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚7970378 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 NGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚65 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚4295 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚257562020085760 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚261386146568192 NADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚59053056 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚228297461246976 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654602240 H@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚665 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚228297528023040 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 2427288000512 D=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚60 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 2295461795840 I@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚4876 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 831258624 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚4293 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚257562003046400 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚322 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚4293 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 5368709120 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 125829120 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚67038476 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚29855042 BDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 K>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59053056 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 4654602240 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚259090667667456 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚261386146568192 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 2295419847680 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚46132224 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 5494538240 _step‚781 DDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521206 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚35153212 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚67038476 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚261386087515136 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 4552982528 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚31880971 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚429 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚223 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚7970443 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 E5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5264 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 122905600 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚61478912 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚259852699366923 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚67034181 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17104896 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚35153210 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚262 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚209 IBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚60 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚262 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 2295419847680 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 5494538240 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚259090667667456 G@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚60 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 P6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚322 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 2290637933095 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 4552982528 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚67034181 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 NADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚58387428 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 4517085184 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚259090684772352 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 2295461795840 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚41948160 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚35153212 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚322 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚31880971 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚41948160 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 122905600 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚20643840 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚41348068 E5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5264 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 122905600 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚61478912 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚259852699366923 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚67034181 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17104896 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚35153210 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚262 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚209 IBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚60 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚262 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 2295419847680 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 5494538240 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚259090667667456 G@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚60 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 P6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚322 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 2290637933095 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 4552982528 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚67034181 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 NADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚58387428 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 4517085184 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚259090684772352 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 2295461795840 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚41948160 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚35153212 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚322 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚31880971 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚41948160 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 122905600 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚20643840 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚41348068 E6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚322 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚7970378 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 NGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚65 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚4295 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚257562020085760 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚261386146568192 NADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚59053056 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚228297461246976 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654602240 H@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚665 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚228297528023040 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 2427288000512 D=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚60 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 2295461795840 I@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚4876 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 831258624 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚4293 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚257562003046400 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚322 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚4293 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 5368709120 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 125829120 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚67038476 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚29855042 BDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 K>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59053056 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 4654602240 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚259090667667456 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚261386146568192 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 2295419847680 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚46132224 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 5494538240 _step‚781 DDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521206 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚35153212 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚67038476 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚261386087515136 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 4552982528 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚31880971 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚429 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚223 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚7970443 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 EDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚41348020 D;DeviceStatsMoQŠ?65nitor.on_train_batch_start/allocation.all.peak‚5264 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚35327309 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚30003166 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 4654602240 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 2306817489920 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 2439320519680 PCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚58387380 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 5368709120 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 125829120 G>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4876 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 4654602240 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17104896 IDeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4289 JBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚665 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚32042786 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚229438819224576 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚260380479324160 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚21992711 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚226999561289728 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚32042786 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 2306775544832 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚32047073 KBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚4876 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚8010457 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚41945088 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 2306775544832 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚262687237764096 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚20643840 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 831258624 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚262687237764096 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 845178880 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 5368709120 trainer/global_step‚9549 F>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 M@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚59049984 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 4552982528 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 122905600 PIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚69 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚261147076100755 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 125829120 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚21992709 E8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚67370093 PCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚59049984 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚67374382 IBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚71 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 4517085184 BDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚41348020 D;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚5264 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚35327309 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚30003166 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 4654602240 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 2306817489920 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 2439320519680 PCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚58387380 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 5368709120 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 125829120 G>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4876 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 4654602240 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17104896 IDeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4289 JBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚665 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚32042786 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚229438819224576 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚260380479324160 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚21992711 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚226999561289728 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚32042786 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 2306775544832 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚32047073 KBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚4876 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚8010457 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚41945088 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 2306775544832 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚262687237764096 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚20643840 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 831258624 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚262687237764096 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 845178880 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 5368709120 trainer/global_step‚9549 F>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 M@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚59049984 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 4552982528 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 122905600 PIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚69 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚261147076100755 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 125829120 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚21992709 E8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚67370093 PCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚59049984 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚67374382 IBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚71 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 4517085184 BDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59053056 NADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚59053056 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚35330860 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 trainer/global_step‚9549 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚35330862 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚4293 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 NADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚58387428 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 5494538240 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17104896 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 4552982528 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚61478912 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚261173550854331 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚322 DDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚30006192 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚322 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 D=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚60 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚260406784557056 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚322 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚32046073 NGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚80 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚227022608662528 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 125829120 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚322 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚262 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚262713791170560 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚35330862 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚258871309762560 _runtime‚2551.234974421 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚67376933 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚21994916 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚8011278 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚20643840 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 5368709120 >5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5264 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚35330860 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654602240 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚32050366 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 2302241091771 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚223 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚4295 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚67376933 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 831258624 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 $ _timestamp‚1.786661202704506e+09 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 122306526 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚67381228 EDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 4654602240 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 4517085184 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚30006274 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 122905600 IBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚60 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚260406801661952 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚229462180820992 SFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚73067520 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚227022588018688 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 2307006613504 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 2439519734784 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 B9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚5264 H@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚665 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 5494538240 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 125829120 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚262713850223616 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚429 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚229462107753472 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 2439572158464 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚32046073 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 122905600 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚4293 G@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚60 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 5368709120 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 125829120 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 2307048561664 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚41948160 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚67381228 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521206 _step‚785 EDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 122905600 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚4293 G@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚60 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 5368709120 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 125829120 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 2307048561664 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚41948160 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚67381228 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521206 _step‚785 EDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59053056 NADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚59053056 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚35330860 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 trainer/global_step‚9549 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚35330862 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚4293 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 NADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚58387428 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 5494538240 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17104896 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 4552982528 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚61478912 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚261173550854331 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚322 DDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚30006192 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚322 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 D=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚60 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚260406784557056 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚322 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚32046073 NGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚80 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚227022608662528 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 125829120 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚322 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚262 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚262713791170560 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚35330862 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚258871309762560 _runtime‚2551.234974421§‚sƒÝ G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚67376933 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚21994916 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚8011278 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚20643840 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 5368709120 >5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5264 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚35330860 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654602240 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚32050366 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 2302241091771 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚223 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚4295 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚67376933 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 831258624 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 $ _timestamp‚1.786661202704506e+09 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 122306526 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚67381228 EDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 4654602240 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 4517085184 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚30006274 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 122905600 IBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚60 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚260406801661952 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚229462180820992 SFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚73067520 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚227022588018688 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 2307006613504 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 2439519734784 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 B9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚5264 H@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚665‘‰gÉO2³- _wandbruntime‚2551  _runtime‚2551šØÈ{s´í 0loss/FRt2/assignment_loss‚0.6764551861686663 /loss/FRt1/detection_loss‚0.5032380193255332 &loss/total_loss‚2.4628953046880344 % _timestamp‚1.7866612027062204e+09 _runtime‚2551.236068938 0loss/FRt1/assignment_loss‚0.7388857247708558 /loss/FRt2/detection_loss‚0.5443163744229789 epoch‚23 trainer/global_step‚9549 _step‚786vœuüxµò epoch‚23 trainer/global_step‚9549 _step‚786 0loss/FRt2/assignment_loss‚0.6764551861686663 /loss/FRt1/detection_loss‚0.5032380193255332 &loss/total_loss‚2.4628953046880344 % _timestamp‚1.7866612027062204e+09 _runtime‚2551.236068938 0loss/FRt1/assignment_loss‚0.7388857247708558 /loss/FRt2/detection_loss‚0.5443163744229789’•8‘ä2¶- _wandbruntime‚2563  _runtime‚2563sˆ¨#Ç·Á & lr-AdamW/pg1‚0.0008221369270144017 & lr-AdamW/pg2‚0.0008221369270144017 trainer/global_step‚9599 % _timestamp‚1.7866612150507479e+09 _runtime‚2563.580761974 _step‚787 ëFCÌ¸Æ % _timestamp‚1.7866612150507479e+09 _runtime‚2563.580761974 _step‚787 & lr-AdamW/pg1‚0.0008221369270144017 & lr-AdamW/pg2‚0.0008221369270144017 trainer/global_step‚9599“p”_2¹- _wandbruntime‚2563  _runtime‚2563Üc¡?¼'º¶O trainer/global_step‚9599 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚4287 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚228151909285888 PCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚58387380 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 4654602240 IBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚67 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 831258624 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 5494538240 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚4287 PCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚59049984 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚35504959 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚261696579108864 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚35504957 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚223 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 122306526 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚32207888 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 2318404255744 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚32212175 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚32207888 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚22102959 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 5494538240 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 % _timestamp‚1.7866612150516758e+09 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚261696596213760 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚322 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17104896 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚30154807 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 2451555268608 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 5494538240 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 5368709120 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚264014941419520 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 122905600 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚20643840 JBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚665 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚35504957 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚429 _step‚788 IDeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4289 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚230603464554496 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚260154396606464 @7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚5264 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚35504959 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚261696579108864 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 4517085184 KDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚60 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 2318404255744 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 2313530981699 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚264014941419520 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 4628521206 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 2313572329719 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 125829120 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚46135296 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚61478912 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 125829120 IDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 M@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚59049984 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 2318362310656 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚261696596213760 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚41945088 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 125829120 G>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4876 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚8051913 PIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚65 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚264015000469504 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17104896 IBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚60 E8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚67712845 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 4654602240 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 4552982528 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚30154874 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚4289 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚264015000469504 UHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚66779136 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚41348020 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0w`Á'»»O UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 5368709120 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚262 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 2318362310656 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 845178880 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚41945088 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚262467985975543 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚262467927588163 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚322 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚262 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚67717134 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4289 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚230603464554496 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚260154396606464 @7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚5264 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚35504959 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚261696579108864 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 4517085184 KDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚60 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 2318404255744 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 2313530981699 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚264014941419520 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 4628521206 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 2313572329719 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 125829120 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚46135296 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚61478912 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 125829120 IDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 M@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚59049984 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 2318362310656 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚261696596213760 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚41945088 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 125829120 G>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4876 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚8051913 PIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚65 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚264015000469504 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17104896 IBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚60 E8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚67712845 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 4654602240 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 4552982528 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚30154874 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚4289 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚264015000469504 UHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚66779136 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚41348020 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 trainer/global_step‚9599 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚4287 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚228151909285888 PCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚58387380 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 4654602240 IBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚67 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 831258624 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 5494538240 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚4287 PCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚59049984 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚35504959 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚261696579108864 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚35504957 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚223 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 122306526 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚32207888 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 2318404255744 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚32212175 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚32207888 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚22102959 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 5494538240 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 % _timestamp‚1.7866612150516758e+09 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚261696596213760 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚322 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17104896 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚30154807 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 2451555268608 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 5494538240 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 5368709120 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚264014941419520 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 122905600 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚20643840 JBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚665 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚35504957 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚429 _step‚788 IDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 2318593379328 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚32215468 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚209 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚41348068 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚30157894 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚67723980 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚67723980 H@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚665 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚44035072 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 NADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚58387428 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 5494538240 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚4293 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚262 PDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚30157825 IBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚60 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 122306526 _runtime‚2563.755048675 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 trainer/global_step‚9599 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚264041553879040 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 DDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59053056 B9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚5264 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 5368709120 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚322 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 2313802902379 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 5368709120 % _timestamp‚1.7866612152247179e+09 I@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚4876 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚32215468 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 122905600 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚264041494825984 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 4552982528 REDeviceÌÕt·» StatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚35508512 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚228174977302528 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚228174956658688 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚262494343954311 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚32211175 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚4293 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚262 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚261722918551552 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚61478912 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 4517085184 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17104896 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚260180599439360 _step‚789 G@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚69 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚230626759579648 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 5494538240 D=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚60 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚261722901446656 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 831258624 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521206 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚322 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 2318593379328 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚264041553879040 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654602240 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 2451802920960 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 125829120 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚262 >5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5264 E6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚322 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚260180582400000¨oG4Ì&¾ÆM _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚228174956658688 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚262494343954311 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚32211175 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚4293 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚262 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚261722918551552 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚61478912 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 4517085184 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17104896 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚260180599439360 _step‚789 G@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚69 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚230626759579648 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 5494538240 D=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚60 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚261722901446656 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 831258624 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521206 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚322 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 2318593379328 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚264041553879040 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654602240 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 2451802920960 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 125829120 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚262 >5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5264 E6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚322 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚260180582400000 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚67719685 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚41948160 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚20643840 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 2451846956032 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚322 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 2318635327488 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚223 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 2313761554311 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 5368709120 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚35508512 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚261722918551552 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚32211175 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 845178880 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚8052728 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚8052661 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 G@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚60 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 4654602240 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚67719685 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 125829120 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 2318593379328 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚32215468 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚209 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚41348068 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚30157894 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚67723980 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚67723980 H@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚665 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚44035072 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 NADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚58387428 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 5494538240 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚4293 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚262 PDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚30157825 IBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚60 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 122306526 _runtime‚2563.755048675 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 trainer/global_step‚9599 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚264041553879040 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 DDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59053056 B9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚5264 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 5368709120 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚322 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 2313802902379 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 5368709120 % _timestamp‚1.7866612152247179e+09 I@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚4876 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚32215468 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 122905600 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚264041494825984 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 4552982528 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚35508512 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚228174977302528•˜z.Ê2¿- _wandbruntime‚2563  _runtime‚25631‘*tÀî _runtime‚2563.755515917 0loss/FRt1/detection_loss‚0.49908264469588814 /loss/FRt2/detection_loss‚0.5356507312404337 _step‚790 0loss/FRt1/assignment_loss‚0.6975798166970519 0loss/FRt2/assignment_loss‚0.6772582704234484 &loss/total_loss‚2.4095714630568223 epoch‚23 trainer/global_step‚9599 % _timestamp‚1.7866612152257035e+09ò[yyÁó 0loss/FRt1/assignment_loss‚0.6975798166970519 0loss/FRt2/assignment_loss‚0.6772582704234484 &loss/total_loss‚2.4095714630568223 epoch‚23 trainer/global_step‚9599 % _timestamp‚1.7866612152257035e+09 _runtime‚2563.755515917 0loss/FRt1/detection_loss‚0.49908264469588814 /loss/FRt2/detection_loss‚0.5356507312404337 _step‚790–HÚ]•Â‚(:Š à’ùÓþù”á gpu.0.gpu‚50.0 gpu.0.memory‚6, gpu.0.memoryAllocated‚7.0313262939453125) gpu.0.memoryAllocatedBytes‚ 6039863296 gpu.0.temp‚35.0 gpu.0.powerWatts‚205.084' gpu.0.enforcedPowerLimitWatts‚500.0 gpu.0.powerPercent‚41.0168 gpu.0.smClock‚1410 gpu.0.memoryClock‚1593$ gpu.0.correctedMemoryErrors‚1955# gpu.0.uncorrectedMemoryErrors‚0Ù¬  Ã‚(:• à’ùÓîÚ‚ä# disk.dm-0.out‚55963.72802734375 cpu‚27.14935828430143 network.recv‚ 4.888531e+06$ disk./.usageGB‚487.4099426269531 disk.sda3.in‚ 24398.3515625( disk.nvme0c0n1.out‚55963.72802734375 disk.sda.in‚ 24398.3515625 disk.sda1.out‚0 disk.sda2.out‚0" disk.nvme0n1.in‚43515.57421875* disk./.usagePercent‚32.710490213876476$ disk.nvme0c0n1.in‚43515.57421875 disk.sda1.in‚0 disk.sda2.in‚0& disk.nvme0n1.out‚55963.72802734375 disk.dm-1.in‚ 24398.3515625 disk.dm-1.out‚ 48.23046875 disk.drbd2165.in‚0 disk.drbd2165.out‚0$ memory_percent‚67.55516529083252) proc.memory.availableMB‚ 5315.76171875# proc.memory.rssMB‚ 2315.30859375* proc.memory.percent‚14.131522178649902 proc.cpu.threads‚17 network.sent‚ 1.2549412e+07 disk.sda3.out‚ 48.23046875 disk.sda.out‚ 48.23046875 disk.dm-0.in‚43515.57421875#0Më2Ä- _wandbruntime‚2569  _runtime‚2569ô¦¨1ÞÅØ 0Purity/*FRt/event_purity‚0.46478983759880066 Purity/0FRt/event_purity‚1 /Purity/1FRt/event_purity‚0.4678390622138977 0Purity/2FRt/event_purity‚0.41697388887405396 +validation_accuracy‚0.21113106608390808 2Purity/*FRt/event_proportion‚0.669805645942688 -Purity/1FRt/FRt_purity‚0.4678390622138977 5validation_average_jet_accuracy‚0.316931813955307 *loss/val_total_loss‚2.4626004440580522 3Purity/0FRt/event_proportion‚0.3301943838596344 /particle/accuracy_2_of_2‚0.6329997181892395 (particle/accuracy‚0.6460774540901184 _step‚791 +jet/accuracy_2_of_2‚0.21113106608390808 % _timestamp‚1.7866612205353541e+09 3Purity/1FRt/event_proportion‚0.6297432780265808 /particle/accuracy_1_of_2‚0.8943846821784973 *jet/accuracy_1_of_2‚0.5816300511360168 +particle/specificity‚0.6748126745223999 +particle/sensitivity‚0.5938393473625183 'particle/f_score‚0.5435607433319092 trainer/global_step‚9599 _runtime‚2569.065355146 -Purity/*FRt/FRt_purity‚0.4784846007823944 2Purity/2FRt/event_proportion‚0.040062315762043 -Purity/2FRt/FRt_purity‚0.5621324777603149 +jet/accuracy_1_of_1‚0.31190961599349976 particle/accuracy_1_of_1‚1 epoch‚23_âãÆÝ +particle/sensitivity‚0.5938393473625183 'particle/f_score‚0.5435607433319092 trainer/global_step‚9599 _runtime‚2569.065355146 -Purity/*FRt/FRt_purity‚0.4784846007823944 2Purity/2FRt/event_proportion‚0.040062315762043 -Purity/2FRt/FRt_purity‚0.5621324777603149 +jet/accuracy_1_of_1‚0.31190961599349976 particle/accuracy_1_of_1‚1 epoch‚23 0Purity/*FRt/event_purity‚0.46478983759880066 Purity/0FRt/event_purity‚1 /Purity/1FRt/event_purity‚0.4678390622138977 0Purity/2FRt/event_purity‚0.41697388887405396 +validation_accuracy‚0.21113106608390808 2Purity/*FRt/event_proportion‚0.669805645942688 -Purity/1FRt/FRt_purity‚0.4678390622138977 5validation_average_jet_accuracy‚0.316931813955307 *loss/val_total_loss‚2.4626004440580522 3Purity/0FRt/event_proportion‚0.3301943838596344 /particle/accuracy_2_of_2‚0.6329997181892395 (particle/accuracy‚0.6460774540901184 _step‚791 +jet/accuracy_2_of_2‚0.21113106608390808 % _timestamp‚1.7866612205353541e+09 3Purity/1FRt/event_proportion‚0.6297432780265808 /particle/accuracy_1_of_2‚0.8943846821784973 *jet/accuracy_1_of_2‚0.5816300511360168 +particle/specificity‚0.6748126745223999—&Dì•Ç‚(:Š ï’ùÓÜ¥­á gpu.0.gpu‚56.0 gpu.0.memory‚11, gpu.0.memoryAllocated‚7.0313262939453125) gpu.0.memoryAllocatedBytes‚ 6039863296 gpu.0.temp‚36.0 gpu.0.powerWatts‚212.315' gpu.0.enforcedPowerLimitWatts‚500.0 gpu.0.powerPercent‚42.463 gpu.0.smClock‚1410 gpu.0.memoryClock‚1593$ gpu.0.correctedMemoryErrors‚1955# gpu.0.uncorrectedMemoryErrors‚0àÉ__È‚(:„ ï’ùÓÕºÅã disk.sda.in‚ 24398.3515625 disk.sda1.in‚0 disk.sda1.out‚0 disk.sda2.out‚0 disk.dm-1.in‚ 24398.3515625 disk.drbd2165.in‚0$ disk./.usageGB‚487.4109420776367$ disk.nvme0c0n1.in‚43515.57421875 disk.drbd2165.out‚0 disk.dm-0.in‚43515.57421875$ memory_percent‚67.79141426086426( proc.memory.availableMB‚ 5277.0546875* proc.memory.percent‚14.369940757751465 cpu‚27.15739637438658 network.sent‚ 1.2637707e+07 disk.sda.out‚48.4375" disk.nvme0n1.in‚43515.57421875% disk.nvme0n1.out‚55966.8232421875# proc.memory.rssMB‚ 2354.37109375 proc.cpu.threads‚17 network.recv‚ 4.913634e+06 disk.sda3.in‚ 24398.3515625' disk.nvme0c0n1.out‚55966.8232421875 disk.sda2.in‚0 disk.dm-1.out‚48.4375" disk.dm-0.out‚55966.8232421875) disk./.usagePercent‚32.71055728785045 disk.sda3.out‚48.4375»MK2É- _wandbruntime‚2581  _runtime‚2581eñÇÊÁ trainer/global_step‚9649 % _timestamp‚1.7866612329560366e+09 _runtime‚2581.486062324 _step‚792 & lr-AdamW/pg1‚0.0008161518415737089 & lr-AdamW/pg2‚0.0008161518415737089#ÌËÆ trainer/global_step‚9649 % _timestamp‚1.7866612329560366e+09 _runtime‚2581.486062324 _step‚792 & lr-AdamW/pg1‚0.0008161518415737089 & lr-AdamW/pg2‚0.0008161518415737089˜ƒ"æ”2Ì- _wandbruntime‚2581  _runtime‚2581oY§½'Í·O WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17104896 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚263403633213440 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 122905600 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚35743213 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 4552982528 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚265737709063680 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 4628521206 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 M@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚59049984 _step‚793 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚263403650318336 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚68142679 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚8101544 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 125829120 IBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚60 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 4654602240 IBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚66 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 2334058745344 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚429 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 IDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 2334058745344 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚35743215 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚223 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚229603548233728 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚322 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚322 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚264173802926892 KDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚60 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚263403633213440 JBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚665 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚30355433 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 2467967806976 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 122905600 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17104896 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚261844585775104 C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚4289 IDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4876 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 4517085184 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 2329158764408 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚61478912 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚265737709063680 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 5368709120 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚322 F?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚60 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 4654602240 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4289 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚20643840 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 D;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚5264 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚262 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚263403650318336 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚8101608 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚232071536684544 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 831258624 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 E8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚68138390 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 2334016800256 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚4287 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚22253825 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚209 PCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚58387380 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚264173744539512 trainer/global_step‚9649 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚265737650013696 _runtime‚2581.490901736 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 125829120 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 125829120 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚35743213 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚22253823 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚261844602814464 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 5368709120 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚262 @7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚5264 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚35743215 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚44038144,´°Ü μO SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 4552982528 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚265737709063680 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 4628521206 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 M@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚59049984 _step‚793 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚263403650318336 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚68142679 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚8101544 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 125829120 IBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚60 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 4654602240 IBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚66 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 2334058745344 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚429 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 IDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 ^LDeviceStatsMonitor.on_train_batch_staÇK6Çært/allocated_bytes.small_pool.allocated‚ 2334058745344 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚35743215 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚223 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚229603548233728 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚322 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚322 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚264173802926892 KDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚60 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚263403633213440 JBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚665 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚30355433 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 2467967806976 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 122905600 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17104896 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚261844585775104 C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚4289 IDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4876 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 4517085184 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 2329158764408 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚61478912 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚265737709063680 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 5368709120 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚322 F?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚60 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 4654602240 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4289 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚20643840 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 D;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚5264 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚262 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚263403650318336 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚8101608 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚232071536684544 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 831258624 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 E8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚68138390 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 2334016800256 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚4287 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚22253825 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚209 PCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚58387380 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚264173744539512 trainer/global_step‚9649 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚265737650013696 _runtime‚2581.490901736 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 125829120 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 125829120 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚35743213 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚22253823 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚261844602814464 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 5368709120 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚262 @7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚5264 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚35743215 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚44038144 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17104896 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚263403633213440 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 122905600 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚35743213 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0™÷UÞ2Ï- _wandbruntime‚2581  _runtime‚2581›`ÓÆ&ÐÀM D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚68149525 PDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521206 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 5368709120 _step‚794 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 4552982528 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 122905600 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚68145230 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚229626595606528 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 2468170536448 trainer/global_step‚9649 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚4293 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 2334289817088 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚41948160 BDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚30358338 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚209 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 831258624 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚41948160 H@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚665 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 >5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5264 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚35746768 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚22256028 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚322 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 $ _timestamp‚1.786661233128506e+09 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚263429955551232 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚68149525 NADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚58387428 EDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 D6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚322 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚32398464 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17104896 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 NGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚65 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 845178880 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚41937920 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 5494538240 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 4552982528 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚223 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚263429972656128 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚261870788608000 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚35746766 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654602240 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚263429955551232 G@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚67 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚429 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚264200160905660 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚262 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚32402757 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚265764262473216 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 2334289817088 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 2468212474368 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚262 IBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚60 K>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59053056 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚264200219293088 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚4295 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚22256030 _runtime‚2581.658948428 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚265764262473216 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚8102375p2qêË&ÑÅM WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 125829120 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚322 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚265764203420160 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚35746766 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚32398464 SFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚62581760 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 2329430685088 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 122905600 I@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚4876 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 5494538240 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17104896 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚61478912 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 5368709120 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 125829120 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚35746768 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚261870771568640 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚322 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚232094766142976 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 5368709120 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚32402757 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚30358405 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚30358338 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚209 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 831258624 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚41948160 H@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚665 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 >5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5264 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚35746768 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚22256028 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚322 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 $ _timestamp‚1.786661233128506e+09 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚263429955551232 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚68149525 NADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚58387428 EDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 D6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚322 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚32398464 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17104896 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 NGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚65 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 845178880 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚41937920 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 5494538240 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 4552982528 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚223 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚263429972656128 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚261870788608000 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚35746766 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654602240 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚263429955551232 G@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚67 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚429 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚264200160905660 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚262 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚32402757 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚265764262473216 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 2334289817088 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 2468212474368 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚262 IBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚60 K>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59053056 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚264200219293088 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚4295 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚22256030 _runtime‚2581.658948428 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚265764262473216 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚8102375 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚68149525 PDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521206 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 5368709120 _step‚794 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 4552982528 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 122905600 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚68145230 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚229626595606528 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 2468170536448 trainer/global_step‚9649 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚4293 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 2334289817088 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚41948160 BÞ‹2Ø- _wandbruntime‚2594  _runtime‚2594µ%_óÙ¶O GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 % _timestamp‚1.7866612455049493e+09 IDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚267065353669120 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚223 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚8142502 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 5494538240 _runtime‚ 2594.03886119 PIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚63 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚209 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 2340720575016 F?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚60 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚35920863 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚265494596026920 KDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚60 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 2480261705728 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 IBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚65 C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚4289 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚264719767207936 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚267065412719104 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 4654602240 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 2345645511168 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 5494538240 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 @7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚5264 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚35920865 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚32564566 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚267065412719104 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚61478912 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 2340761923036 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚264719750103040 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 4552982528 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 2345603566080 JBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚665 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚30506640 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚233236199223296 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 D;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚5264 KBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚4876 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 5368709120 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚262 IBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚60 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 4654602240 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 5368709120 _step‚797 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚32564566 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚2307559167öïQÉ873728 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚263153875451904 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 5494538240 E8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚68481142 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚267065353669120 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚41945088 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 5368709120 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 122905600 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚32560279 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚264719750103040 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 4552982528 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚44038144 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚22364075 F>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚8142565 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚20643840 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4289 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚4287 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚68485431 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚35920865 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚233236134541312 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚262 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 125829120 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚262 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚35920863 G>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4876 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚41945088 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚4287 UHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚64681984 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17104896 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 845178880 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚41348020 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 122905600 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚30506575 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚230755937517568 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 4628521206 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚263153892491264 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 PCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚59049984 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 125829120 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚322 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚22364073 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚265494654414300 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 4517085184 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 122306526 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚32560279 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚264719767207936 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚429 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚322 BDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚267065353669120 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚223 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚8142502 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 5494538240 _runtime‚ 2594.03886119 PIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚63 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚209 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 2340720575016 F?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚60 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚35920863 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚265494596026920 KDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚60 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 2480261705728 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 IBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚65 C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚4289 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚264719767207936 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚267065412719104 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 4654602240 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 2345645511168 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 5494538240 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 @7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚5264 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚35920865 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚32564566 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚267065412719104 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚61478912 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 2340761923036 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚264719750103040 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 4552982528 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 2345603566080 JBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚665 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚30506640 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚233236199223296 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 D;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚5264 KBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚4876 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 5368709120 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚262 IBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚60 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 4654602240 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 5368709120 _step‚797 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚32564566 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚230755916873728 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚263153875451904 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 5494538240 E8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚68481142 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚267065353669120 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚41945088 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 5368709120 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 122905600 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚32560279 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚264719750103040 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 4552982528 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚44038144 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚22364075 F>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚8142565 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚20643840 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4289 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚4287 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚68485431 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚35920865 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚233236134541312 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚262 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 125829120 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚262 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚35920863 G>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4876 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚41945088 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚4287 UHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚64681984 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17104896 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 845178880 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚41348020 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 122905600 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0J©Ë2Û- _wandbruntime‚2594  _runtime‚2594#PýüÇ&ÜÁM REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚41948160 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚41348068 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 5494538240 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 5494538240 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚264746089545728 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚41948160 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 2345834634752 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 2345876582912 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚32563566 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚263180061245440 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 5368709120 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚68487982 NADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚58387428 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚35924416 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚32567859 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 2480507945984 G@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚60 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚68492277 K>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59053056 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17104896 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚35924418 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚4293 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚8143328 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚263180078284800 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚322 >5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5264 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17104896 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 4552982528 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚267091966128640 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 2480463910912 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚265521012393068 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521206 NADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚59053056 SFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚64678912 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚20643840 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 2345834634752 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 4517085184 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚322 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚262 IBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚60 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚223 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚32563566 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 5368709120 D=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚60 E8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 2340992495696 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 2340951147628 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 122306526 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 125829120 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 P6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚322 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚262 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚264746072440832 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚68492277 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚30509542 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚22366278 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚265521070780496 D6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚322 EDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654602240 I@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚4876 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚30509608 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚267091907075584 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚4293 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 125829120 trainer/global_step‚9699 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚35924416 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚429 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚262 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 4654602240 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚233259492836352 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚44035072 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360]wVØZÝÆM A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚4295 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 122905600 G@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚66 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚209 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚230778964246528 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 5368709120 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚264746072440832 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654602240 I@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚4876 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚30509608 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚267091907075584 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚4293 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 125829120 trainer/global_step‚9699 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚35924416 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚429 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚262 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 4654602240 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚233259492836352 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚44035072 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚41948160 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚41348068 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 5494538240 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 5494538240 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚264746089545728 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚41948160 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 2345834634752 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 2345876582912 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚32563566 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚263180061245440 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 5368709120 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚68487982 NADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚58387428 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚35924416 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚32567859 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 2480507945984 G@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚60 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚68492277 K>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59053056 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17104896 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚35924418 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚4293 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚8143328 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚263180078284800 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚322 >5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5264 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17104896 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 4552982528 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚267091966128640 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 2480463910912 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚265521012393068 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521206 NADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚59053056 SFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚64678912 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚20643840 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 2345834634752 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 4517085184 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚322 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚262 IBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚60 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚223 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚32563566 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 5368709120 D=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚60 E8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 2340992495696 XFDeviceStatsMonitor.on_train_batch_end/requ]äØr ested_bytes.small_pool.freed‚ 2340951147628 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 122306526 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 125829120 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 P6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚322 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚262 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚264746072440832 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚68492277 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚30509542 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚22366278 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚265521070780496 D6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚322 EDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚268393057324544 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4289 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚209 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 2492493476864 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚322 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 F>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17104896 PCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚59049984 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 2357232276992 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚41348020 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 125829120 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 5368709120 _runtime‚2606.486882342 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚266035884097536 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚20643840 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 4628521206 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 125829120 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚234400778990592 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚231908306157568 C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚4289 M@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚59049984 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 2352323733644 G>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4876 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 122905600 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚30657656 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚262 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚36098513 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 2352282385624 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚41945088 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚8183269 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚36098515 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚4287 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 125829120 @7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚5264 PIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚62 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚61478912 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚266815505901708 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 5494538240 IDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 IBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚60 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚266035884097536 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚268393116374528 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 4654602240 IDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 IBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚60 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚266035884097536 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚268393116374528 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 4654602240 IDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚268393057324544 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4289 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚209 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 2492493476864 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚322 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 F>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17104896 PCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚59049984 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 2357232276992 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚41348020 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 125829120 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 5368709120 _runtime‚2606.486882342 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚266035884097536 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚20643840 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 4628521206 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 125829120 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚234400778990592 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚231908306157568 C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚4289 M@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚59049984 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 2352323733644 G>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4876 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 122905600 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚30657656 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚262 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚36098513 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 2352282385624 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚41945088 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚8183269 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚36098515 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚4287 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 125829120 @7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚5264 PIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚62 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚61478912 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚266815505901708 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 5494538240 I6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚322 IBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚60 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 2357463348736 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚41948160 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚268419669784064 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚209 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚30660611 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚234424136269824 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚68835029 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 5368709120 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17104896 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 122905600 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 845178880 E5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5264 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚266062206435328 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚36102066 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚322 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚22476528 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚4293 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚268419669784064 PDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521206 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚264489367961600 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚322 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 4552982528 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚8184083 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 EDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59053056 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654602240 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 2357421400576 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚61478912 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 E?DeviceStatsMonitor.on_train_batch_eiñƒÔŒ nd/oversize_allocations.peak‚0 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 5368709120 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 I@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚4876 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚41948160 trainer/global_step‚9749 _runtime‚2606.648272724 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚32732961 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚429 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 122306526 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚262 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚262 G@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚60 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚266062206435328 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚223 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 H@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚665 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 2492782739456 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚20643840 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚68835029 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 831258624 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚322 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚32732961 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 2357463348736 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 5494538240 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚36102068 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 4654602240 NGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚56 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 125829120 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚266062189330432 B9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚5264 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚30660669 _step‚802 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚68830734 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚36102068 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 5494538240 $ _timestamp‚1.786661258117931e+09 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 4552982528 NADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚58387428 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 2352554306304p¢ëË&ëÅM S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚266841863880476 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521206 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚264489367961600 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚322 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 4552982528 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚8184083 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 EDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59053056 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654602240 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 2357421400576 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚61478912 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 5368709120 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 I@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚4876 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚41948160 trainer/global_step‚9749 _runtime‚2606.648272724 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚32732961 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚429 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 122306526 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚262 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚262 G@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚60 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚266062206435328 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚223 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 H@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚665 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 2492782739456 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚20643840 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚68835029 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 831258624 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚322 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚32732961 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 2357463348736 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 5494538240 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚36102068 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 4654602240 NGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚56 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 125829120 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚266062189330432 B9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚5264 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚30660669 _step‚802 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚68830734 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚36102068 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 5494538240 $ _timestamp‚1.786661258117931e+09 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 4552982528 NADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚58387428 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 2352554306304 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚264489350922240 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚268419610731008 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚32728668 SFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚64678912 B6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚322 IBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚60 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 2357463348736 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚41948160 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚268419669784064 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚209 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚30660611 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚234424136269824 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚68835029 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 5368709120 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17104896 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 122905600 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 845178880 E5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5264 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚266062206435328 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚36102066 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚322 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚22476528 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚4293 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚268419669784064 PDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚269720760979968 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚36276163 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 4517085184 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚322 trainer/global_step‚9799 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17104896 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 2504819329536 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚44038144 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 2368777097728 KBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚4876 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 IBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚60 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚32894770 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 4552982528 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 4654602240 JBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚665 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 2363885544252 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 F>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 2368819042816 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚36276165 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 2363844196232 PCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚59049984 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚269720760979968 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚41945088 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 IDeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4289 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 E8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚69166646 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17104896 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚22584573 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚36276165 @7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚5264 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 2368819042816 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚41945088 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚267352000987136 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 122905600 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚61478912 BDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4876 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 845178880 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚268136299001736 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚265772454805504 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 5494538240 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚269720820029952 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚267352000987136 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚30808154 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 5368709120 M@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚59049984 UHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚64681984 «ºKö¼O YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚267351983882240 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚235565429445120 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚233060654153728 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚41348020 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 5368709120 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 125829120 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚267351983882240 _runtime‚2619.476894004 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚233060674797568 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚262 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 2504775291392 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚322 % _timestamp‚1.7866612709430969e+09 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚69170935 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚262 KDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚60 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚32890483 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚8223516 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚209 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚235565494127104 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚268136357389116 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 5494538240 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚322 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 4654602240 D;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚5264 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚262 C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚4289 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚269720820029952 PIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚63 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚20643840 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚265772471844864 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 125829120 _step‚805 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚4287 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4289 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 E8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚69166646 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17104896 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚22584573 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚36276165 @7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚5264 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 2368819042816 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚41945088 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚267352000987136 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 122905600 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚61478912 BDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4876 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 845178880 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚Bß*w268136299001736 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚265772454805504 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 5494538240 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚269720820029952 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚267352000987136 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚30808154 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 5368709120 M@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚59049984 UHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚64681984 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚4287 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 122306526 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚269720760979968 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚36276163 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 4517085184 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚322 trainer/global_step‚9799 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17104896 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 2504819329536 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚44038144 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 2368777097728 KBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚4876 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 IBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚60 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚32894770 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 4552982528 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 4654602240 JBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚665 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 2363885544252 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 F>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 2368819042816 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚36276165 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 2363844196232 PCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚59049984 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚269720760979968 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚41945088 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 IDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521206 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 4517085184 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚322 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚41948160 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 _step‚806 NADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚59053056 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚69177781 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 2364116116912 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚269747314386432 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 4654602240 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 122905600 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚36279716 _runtime‚2619.638563399 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚8224355 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚322 G@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚60 K>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59053056 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚46132224 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚262 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚32898063 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 845178880 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 5368709120 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚32898063 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 5368709120 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 2369050114560 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚209 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚267378323324928 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚235588788100608 SFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚66776064 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 2364074768844 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚322 trainer/global_step‚9799 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 IBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚60 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚22586778 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚265798640599040 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 5494538240 EDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚4293 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚267378306220032 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17104896 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚4295 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚22586780 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚8224423 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚69173486 >5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5264 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚32893770 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚269747373439488 G@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚70 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 B9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚5264 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚41348068 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 125829120 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚69177781 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚268162715367884 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 5494538240 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚262 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚36279718 DDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚30811133 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 831258624 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 125829120 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17104896 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654602240 I@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚4876 NGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚68 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚69173486 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚20643840 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 125829120 % _timestamp‚1.7866612711082816e+09 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 4552982528 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 2369050114560 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 2505065930240 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 5494538240â Ì&ùÆM [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚267378323324928 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚235588788100608 SFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚66776064 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 2364074768844 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚322 trainer/global_step‚9799 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 IBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚60 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚22586778 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚265798640599040 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 5494538240 EDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚4293 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚267378306220032 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17104896 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚4295 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚22586780 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚8224423 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚69173486 >5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5264 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚32893770 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚269747373439488 G@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚70 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 B9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚5264 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚41348068 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 125829120 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚69177781 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚268162715367884 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 5494538240 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚262 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚36279718 DDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚30811133 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 831258624 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 125829120 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17104896 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654602240 I@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚4876 NGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚68 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚69173486 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚20643840 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 125829120 % _timestamp‚1.7866612711082816e+09 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 4552982528 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 2369050114560 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 2505065930240 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 5494538240 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚233083722170368 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 BDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521206 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 4517085184 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚322 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚41948160 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 _step‚806 NADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚59053056 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚69177781 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 2364116116912 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚269747314386432 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 4654602240 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 122905600 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚36279716 _runtime‚2619.638563399 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚8224355 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚322 G@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚60 K>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59053056 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚46132224 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚262 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚32898063 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 845178880 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 5368709120 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚32898063 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 5368709120 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 2369050114560 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚209¦•Ò±°2ú- _wandbruntime‚2619  _runtime‚2619ü#±¢rûì 0loss/FRt1/assignment_loss‚0.7440768080115132 0loss/FRt2/assignment_loss‚0.6717431241063009 epoch‚24 % _timestamp‚1.7866612711099548e+09 _step‚807 /loss/FRt1/detection_loss‚0.5479775256655114 /loss/FRt2/detection_loss‚0.4834241505973367 %loss/total_loss‚2.447221608380662 trainer/global_step‚9799 _runtime‚2619.639928975 ‘Êwüñ %loss/total_loss‚2.447221608380662 trainer/global_step‚9799 _runtime‚2619.639928975 0loss/FRt1/assignment_loss‚0.7440768080115132 0loss/FRt2/assignment_loss‚0.6717431241063009 epoch‚24 % _timestamp‚1.7866612711099548e+09 _step‚807 /loss/FRt1/detection_loss‚0.5479775256655114 /loss/FRt2/detection_loss‚0.4834241505973367§JŸË–ý‚(:‹ œ“ùÓ†¿ˆá gpu.0.gpu‚60.0 gpu.0.memory‚11, gpu.0.memoryAllocated‚7.0313262939453125) gpu.0.memoryAllocatedBytes‚ 6039863296 gpu.0.temp‚34.0 gpu.0.powerWatts‚120.418' gpu.0.enforcedPowerLimitWatts‚500.0 gpu.0.powerPercent‚24.0836 gpu.0.smClock‚1410 gpu.0.memoryClock‚1593$ gpu.0.correctedMemoryErrors‚1955# gpu.0.uncorrectedMemoryErrors‚0\/Ûršþ‚(: œ“ùÓ§¼©ã disk.sda3.out‚ 49.01953125 disk.sda.in‚ 24485.015625 disk.sda1.in‚0 disk.sda2.in‚0 disk.sda2.out‚0 disk.dm-1.out‚ 49.01953125# proc.memory.rssMB‚ 2354.37109375 network.sent‚ 1.2858618e+07 disk.sda3.in‚ 24485.015625 disk.sda1.out‚0 disk.dm-1.in‚ 24485.015625# disk.dm-0.out‚56060.97119140625$ memory_percent‚67.79229640960693) proc.memory.availableMB‚ 5276.91015625 proc.cpu.threads‚17 network.recv‚ 5.005371e+06$ disk./.usageGB‚487.4761657714844 disk.drbd2165.in‚0 disk.drbd2165.out‚0 disk.dm-0.in‚ 43522.7265625* proc.memory.percent‚14.369940757751465 cpu‚27.156055044368962# disk.nvme0c0n1.in‚ 43522.7265625( disk.nvme0c0n1.out‚56060.97119140625 disk.sda.out‚ 49.01953125! disk.nvme0n1.in‚ 43522.7265625& disk.nvme0n1.out‚56060.97119140625) disk./.usagePercent‚32.71493450467089jH2ÿ- _wandbruntime‚2632  _runtime‚2632=¦4À½ % lr-AdamW/pg1‚0.000792172859772917 % lr-AdamW/pg2‚0.000792172859772917 trainer/global_step‚9849 # _timestamp‚1.78666128385236e+09 _runtime‚2632.382408718 _step‚808¥(Ñ/È _step‚808 % lr-AdamW/pg1‚0.000792172859772917 % lr-AdamW/pg2‚0.000792172859772917 trainer/global_step‚9849 # _timestamp‚1.78666128385236e+09 _runtime‚2632.382408718¨Àa2™2‚- _wandbruntime‚2632  _runtime‚2632H a(ƒ·O PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚36453815 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 2380363863552 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚234213043437568 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 125829120 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 BDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚271048464635392 PCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚59049984 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚41945088 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 122905600 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4289 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17104896 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 trainer/global_step‚9849 C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚4289 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚271048523685376 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 2375447354860 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚33055585 PCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚58387380 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 5494538240 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17104896 JBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚665 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 5368709120 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚322 _step‚809 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚54523904 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚322 _runtime‚2632.383863564 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚36453813 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 4552982528 IBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚81 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚33059872 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚262 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚268668117876736 G>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4876 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚322 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚269457208876524 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 125829120 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 125829120 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚262 IBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.Zm–Ì•current‚60 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 122905600 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 5368709120 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 2380363863552 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚236730075551232 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚268668100771840 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 2380405808640 UHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚75167744 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚267081761521664 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚4287 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 5494538240 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 F>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 2380405808640 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 122306526 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 5368709120 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚262 IDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚271048464635392 PCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚59049984 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚41945088 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 122905600 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4289 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17104896 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 trainer/global_step‚9849 C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚4289 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚271048523685376 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 2375447354860 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚33055585 PCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚58387380 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 5494538240 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17104896 JBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚665 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 5368709120 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚322 _step‚809 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚54523904 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚322 _runtime‚2632.383863564 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚36453813 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 4552982528 IBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚81 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚33059872 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚262 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚268668117876736 G>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4876 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚322 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚269457208876524 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 125829120 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 125829120 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚262 IBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚60 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 122905600 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 5368709120 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 2380363863552 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚236730075551232 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚268668100771840 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 2380405808640 UHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚75167744 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚267081761521664 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚4287 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 5494538240 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 F>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 2380405808640 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 122306526 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 5368709120 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚262©; Xƒ2…- _wandbruntime‚2632  _runtime‚2632wδ Ç&†ÁM JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚262 PDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 4552982528 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚8265067 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 831258624 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚269483566855292 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 5494538240 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 125829120 trainer/global_step‚9849 >5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5264 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚22697030 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚234236090810368 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚269483625242720 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521206 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚267107930275840 K>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59053056 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 2517347951104 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚41348068 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 125829120 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17104896 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 2380636880384 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚69520533 % _timestamp‚1.7866612840296683e+09 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚4293 G@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚62 BDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 5368709120 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 2380636880384 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 4654602240 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 122905600 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 2380594932224 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 5494538240 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚41948160 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚41948160 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚69516238 NGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚60 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 845178880 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚271075077094912 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚8265007 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 2375636579452 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚236753374082560 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 122306526 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚4293 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚271075018041856 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚61478912 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚30962097 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 2517303916032 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚322 G@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚60 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚33058872 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 NADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚58387428 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚268694423109632 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚22697028 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚44035072 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚322 DDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654602240 EDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚36457366 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚271075077094912 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚209 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚322 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚30962035 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17104896 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚36457368 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚322 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚4295 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚33063165 E8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚269483625242720 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521206 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚267107930275840 K>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59053056 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 2517347951104 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚41348068 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 125829120 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17104896 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 2380636880384 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚69520533 % _timestamp‚1.7866612840296683e+09 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚4293 G@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚62 BDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 5368709120 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 2380636880384 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 4654602240 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 122905600 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 2380594932224 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 5494538240 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚41948160 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚41948160 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚69516238 NGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚60 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 845178880 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚271075077094912 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚8265007 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 2375636579452 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚236753374082560 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 122306526 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚4293 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚271075018041856 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚61478912 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚30962097 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 2517303916032 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚322 G@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚60 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚33058872 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 NADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚58387428 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚268694423109632 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚22697028 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚44035072 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚322 DDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654602240 EDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚36457366 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚271075077094912 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚209 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚322 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚30962035 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17104896 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚36457368 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚322 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚4295 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚33063165 EDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 4552982528 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚8265067 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 831258624 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚269483566855292 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 5494538240 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 125829120 trainer/global_step‚9849 >5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5264ª¹Tý2ˆ- _wandbruntime‚2632  _runtime‚2632ë@er‰ì 0loss/FRt1/assignment_loss‚0.7010003427685738 0loss/FRt1/detection_loss‚0.48449499319158673 trainer/global_step‚9849 $ _timestamp‚1.786661284030775e+09 _step‚811 0loss/FRt2/assignment_loss‚0.6733316494196193 /loss/FRt2/detection_loss‚0.5011294659440012 %loss/total_loss‚2.359956451323781 epoch‚24 _runtime‚2632.560606484.ý«RwŠñ $ _timestamp‚1.786661284030775e+09 _step‚811 0loss/FRt2/assignment_loss‚0.6733316494196193 /loss/FRt2/detection_loss‚0.5011294659440012 %loss/total_loss‚2.359956451323781 epoch‚24 _runtime‚2632.560606484 0loss/FRt1/assignment_loss‚0.7010003427685738 0loss/FRt1/detection_loss‚0.48449499319158673 trainer/global_step‚9849«Ñh‘ï¡‹‚(:– «“ùÓ¤ä§á gpu.0.gpu‚60.0 gpu.0.memory‚11, gpu.0.memoryAllocated‚7.0313262939453125) gpu.0.memoryAllocatedBytes‚ 6039863296 gpu.0.temp‚34.0 gpu.0.powerWatts‚119.843' gpu.0.enforcedPowerLimitWatts‚500.0) gpu.0.powerPercent‚23.968600000000002 gpu.0.smClock‚1410 gpu.0.memoryClock‚1593$ gpu.0.correctedMemoryErrors‚1955# gpu.0.uncorrectedMemoryErrors‚0X_¼ ŽŒ‚(:ƒ «“ùÓ„Žïã$ memory_percent‚67.79420375823975* proc.memory.percent‚14.369940757751465 disk.sda3.out‚ 49.19921875' disk.nvme0c0n1.out‚56064.5498046875 network.sent‚ 1.2923624e+07 disk.drbd2165.in‚0 disk.dm-0.in‚43530.24609375 disk.dm-1.out‚ 49.19921875 proc.cpu.threads‚17$ disk.nvme0c0n1.in‚43530.24609375 disk.sda.in‚25317.75 disk.sda2.in‚0% disk.nvme0n1.out‚56064.5498046875 disk.dm-1.in‚25317.75) proc.memory.availableMB‚ 5276.59765625# proc.memory.rssMB‚ 2354.37109375 cpu‚27.155729571154076 disk.sda.out‚ 49.19921875) disk./.usagePercent‚32.71494116086678$ disk./.usageGB‚487.4762649536133 disk.drbd2165.out‚0" disk.dm-0.out‚56064.5498046875 disk.sda3.in‚25317.75 disk.sda1.out‚0 disk.sda2.out‚0 network.recv‚ 5.027468e+06 disk.sda1.in‚0" disk.nvme0n1.in‚43530.24609375Šî9{2- _wandbruntime‚2645  _runtime‚2645yÏÂÇŽÁ trainer/global_step‚9899 % _timestamp‚1.7866612966150522e+09 _runtime‚2645.145077603 _step‚812 & lr-AdamW/pg1‚0.0007861703794820698 & lr-AdamW/pg2‚0.0007861703794820698¾ôhÌÆ & lr-AdamW/pg2‚0.0007861703794820698 trainer/global_step‚9899 % _timestamp‚1.7866612966150522e+09 _runtime‚2645.145077603 _step‚812 & lr-AdamW/pg1‚0.0007861703794820698¬uÆÊ2- _wandbruntime‚2645  _runtime‚2645¹A´½'‘·O SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 4552982528 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚33220687 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 2387009165468 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 5494538240 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚269984234766336 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 4517085184 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚262 G>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4876 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚20643840 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 5368709120 KDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚60 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚41945088 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 E8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚69852150 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 122905600 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 125829120 BDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 2529359271936 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 4628521206 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚268391034159104 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚36631465 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚36631465 KBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚4876 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚223 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚322 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚269984217661440 PCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚59049984 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚269984217661440 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚4287 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚61478912 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚33220687 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 122905600 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚22805073 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚235365412077568 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚36631463 JBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚665 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚237894750705664 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚270778060363932 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚272376168290816 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 5368709120 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 2529405407232 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚322 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 IDeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4289 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚237894817484800 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚262 _runtime‚2645.149908151 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 845178880 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 831258624 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚33224974 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 5494538240 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 125829120 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚272376168290816 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 4552982528 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚33224974 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17104896 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚22805075 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚8304849 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 2391992574464 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 IBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚60 M@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚59049984 PCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚58387380 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 2391950629376 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 2386967817448 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚31109922 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚268391051198464 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 4654602240 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 2391950629376 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚69856439 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚36631463 UHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚66779136 @7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚5264 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚269984234766336 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚31109992 PIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚68 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 F?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚60 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚272376227340800 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚270778001976552 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 5368709120 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚322 % _timestamp‚1.7866612966160018e+09ç0UÝÂ'’¼O BDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 2529359271936 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 4628521206 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚268391034159104 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚36631465 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚36631465 KBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚4876 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚223 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚322 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚269984217661440 PCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚59049984 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚269984217661440 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚4287 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚61478912 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚33220687 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 122905600 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚22805073 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚235365412077568 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚36631463 JBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚665 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚237894750705664 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚270778060363932 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚272376168290816 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 5368709120 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 2529405407232 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚322 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 IDeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4289 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚237894817484800 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚262 _runtime‚2645.149908151 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 845178880 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 831258624 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚33224974 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 5494538240 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 125829120 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚272376168290816 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 4552982528 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚33224974 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17104896 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚22805075 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚8304849 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 2391992574464 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 IBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚60 M@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚59049984 PCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚58387380 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 2391950629376 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 2386967817448 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚31109922 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚268391051198464 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 4654602240 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 2391950629376 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚69856439 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚36631463 UHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚66779136 @7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚5264 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚269984234766336 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚31109992 PIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚68 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 F?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚60 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚272376227340800 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚270778001976552 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 5368709120 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚322 % _timestamp‚1.7866612966160018e+09 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 4552982528 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚33220687 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 2387009165468 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 5494538240 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚269984234766336 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 4517085184 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚262 G>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4876 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚20643840 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 5368709120 KDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚60 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚41945088 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 E8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚69852150 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 122905600 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 125829120­±S×2“- _wandbruntime‚2645  _runtime‚2645Ô‘4m!”ÁM YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 2392223646208 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 2392181698048 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚69858990 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 5368709120 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚33228267 B9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚5264 _runtime‚2645.311211753 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 5494538240 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 845178880 D=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚60 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 5494538240 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 IBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚60 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚272402780750336 P6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚322 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 5368709120 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 trainer/global_step‚9899 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚270010539999232 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 2392223646208 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚262 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚36635018 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚235388438806528 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚61478912 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚270010539999232 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚31112934 SFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚66776064 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 2387198390060 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚322 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚270804418342700 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 125829120 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17104896 NGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚66 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 _step‚814 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚33228267 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚33223974 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚322 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚4295 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚69858990 DDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654602240 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 4552982528 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚268417236992000 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚268417219952640 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚4293 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚69863285 EDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59053056 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚8305722 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚8305656 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚270804476730128 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 5368709120 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚322 EDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 46Ý# Z28521206 NADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚59053056 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚41348068 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚69863285 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚429 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚262 >5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5264 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 122905600 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 5494538240 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚41948160 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚31113002 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚237918041372672 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 4517085184§ÿ]¦Ì&•ÆM LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚4293 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚69863285 EDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59053056 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚8305722 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚8305656 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚270804476730128 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 5368709120 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚322 EDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521206 NADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚59053056 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚41348068 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚69863285 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚429 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚262 >5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5264 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 122905600 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 5494538240 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚41948160 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚31113002 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚237918041372672 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 4517085184 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 2392223646208 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 2392181698048 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚69858990 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 5368709120 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚33228267 B9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚5264 _runtime‚2645.311211753 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 5494538240 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 845178880 D=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚60 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 5494538240 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 IBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚60 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚272402780750336 P6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚322 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 5368709120 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 trainer/global_step‚9899 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚270010539999232 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 2392223646208 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚262 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚36635018 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚235388438806528 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚61478912 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚270010539999232 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚31112934 SFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚66776064 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 2387198390060 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚322 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚270804418342700 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 125829120 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17104896 NGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚66 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 _step‚814 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚33228267 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚33223974 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚322 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚4295 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚69858990 DDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654602240 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 4552982528 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚268417236992000 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚268417219952640®(ø2–- _wandbruntime‚2645  _runtime‚2645…:¹vr—ì _step‚815 0loss/FRt2/assignment_loss‚0.6811482765468326 /loss/FRt2/detection_loss‚0.5156128017921512 %loss/total_loss‚2.457696218823308 epoch‚24 % _timestamp‚1.7866612967819045e+09 _runtime‚2645.311706998 0loss/FRt1/assignment_loss‚0.7461687605772389 /loss/FRt1/detection_loss‚0.5147663799070852 trainer/global_step‚9899$ß”Aw˜ñ 0loss/FRt2/assignment_loss‚0.6811482765468326 /loss/FRt2/detection_loss‚0.5156128017921512 %loss/total_loss‚2.457696218823308 epoch‚24 % _timestamp‚1.7866612967819045e+09 _runtime‚2645.311706998 0loss/FRt1/assignment_loss‚0.7461687605772389 /loss/FRt1/detection_loss‚0.5147663799070852 trainer/global_step‚9899 _step‚815¯ÀhÆ•™‚(:Š º“ùÓåýþà gpu.0.gpu‚40.0 gpu.0.memory‚9, gpu.0.memoryAllocated‚7.0313262939453125) gpu.0.memoryAllocatedBytes‚ 6039863296 gpu.0.temp‚34.0 gpu.0.powerWatts‚119.694' gpu.0.enforcedPowerLimitWatts‚500.0 gpu.0.powerPercent‚23.9388 gpu.0.smClock‚1410 gpu.0.memoryClock‚1593$ gpu.0.correctedMemoryErrors‚1955# gpu.0.uncorrectedMemoryErrors‚0Y]£~’š‚(:‡ º“ùÓבµã" disk.nvme0n1.in‚43531.49609375& disk.nvme0n1.out‚56112.08837890625 disk.drbd2165.in‚0 disk.drbd2165.out‚0 disk.dm-0.in‚43531.49609375$ disk.nvme0c0n1.in‚43531.49609375( disk.nvme0c0n1.out‚56112.08837890625 disk.sda.in‚25317.75) disk./.usagePercent‚32.71494064885171 disk.dm-1.in‚25317.75# disk.dm-0.out‚56112.08837890625 disk.sda.out‚ 49.60546875 disk.sda1.out‚0 cpu‚27.155521307162164 proc.cpu.threads‚17 disk.sda3.out‚ 49.60546875 network.sent‚ 1.2988764e+07% disk./.usageGB‚487.47625732421875 disk.sda3.in‚25317.75 disk.sda1.in‚0 disk.sda2.in‚0 disk.sda2.out‚0$ memory_percent‚67.79415607452393 network.recv‚ 5.048938e+06 disk.dm-1.out‚ 49.60546875) proc.memory.availableMB‚ 5276.60546875# proc.memory.rssMB‚ 2354.37109375* proc.memory.percent‚14.369940757751465`yµÔ2›- _wandbruntime‚2657  _runtime‚2657cy¸ÇœÁ trainer/global_step‚9949 % _timestamp‚1.7866613090721135e+09 _runtime‚2657.602121075 _step‚816 & lr-AdamW/pg1‚0.0007801655760353383 & lr-AdamW/pg2‚0.0007801655760353383…ûÑÆÌÆ & lr-AdamW/pg1‚0.0007801655760353383 & lr-AdamW/pg2‚0.0007801655760353383 trainer/global_step‚9949 % _timestamp‚1.7866613090721135e+09 _runtime‚2657.602121075 _step‚816°üà 2ž- _wandbruntime‚2657  _runtime‚2657¬”O¼'Ÿ¶O R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚273703871946240 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚70199191 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4289 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚36809115 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚33390076 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚41945088 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚31260721 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 122306526 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚36809115 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚236517760073728 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 2541683741184 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚239059395582464 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚20643840 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚272098911851340 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚36809113 F>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚271300334551040 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 4552982528 IBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚72 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 122905600 PIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚70 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 5494538240 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 5368709120 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 4552982528 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚22915325 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 125829120 @7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚5264 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17104896 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚22915323 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚61478912 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚322 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚41945088 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚209 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 2398570976076 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚322 KDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚60 IBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚60 C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚4289 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 2403579340288 KBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚4876 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 4517085184 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚273703871946240 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 122905600 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 5368709120 M@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚59049984 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚271300351655936 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚239059464458752 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 4628521206 JBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚665 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚223 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚272098853463960 _step‚817 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚4287 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 831258624 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 2403579340288 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚36809113 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚273703930996224 IDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4876 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 4654602240 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17104896 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 845178880 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 5494538240 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚322 IDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚322 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚41945088 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚209 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 2398570976076 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚322 KDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚60 IBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚60 C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚4289 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 2403579340288 KBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚4876 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 4517085184 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚273703871946240 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 122905600 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 5368709120 M@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚59049984 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚271300351655936 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚239059464458752 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 4628521206 JBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚665 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚223 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚272098853463960 _step‚817 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚4287 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 831258624 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 2403579340288 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚36809113 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚273703930996224 IDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4876 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 4654602240 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17104896 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 845178880 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 5494538240 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚322 IDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚273703871946240 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚70199191 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4289 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚36809115 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚33390076 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚41945088 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚31260721 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 122306526 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚36809115 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚23651776007372™)Äõ78 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 2541683741184 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚239059395582464 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚20643840 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚272098911851340 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚36809113 F>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚271300334551040 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 4552982528 IBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚72 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 122905600 PIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚70 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 5494538240 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 5368709120 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 4552982528 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚22915325 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 125829120±ß."Â2¡- _wandbruntime‚2657  _runtime‚2657Ÿ‰^Ç&¢ÁM aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 2541924410880 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚262 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚36812668 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 2403810412032 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚22917530 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚46132224 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 5494538240 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17104896 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17104896 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚36812668 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚4293 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 125829120 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚4295 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚273730484405760 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚273730484405760 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚271326673993728 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 125829120 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚33389076 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 2398801548736 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 5368709120 EDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚31263617 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚236540807446528 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 2541878278656 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚269726526668800 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 5368709120 >5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5264 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654602240 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 2398760200668 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚31263687 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚8346089 G@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚60 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 2403810412032 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 2403768463872 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 831258624 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚22917528 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚36812666 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 5368709120 D=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚60 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚239082685725184 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚20643840 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚70206037 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚33393369 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 845178880 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚272125328217536 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 NGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚68 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚33393369 SFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚66776064 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚41948160 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 4517085184 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚41948160 E6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚322 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚269726509629440 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚41348068 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 5494538240 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚223 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚4293 B9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚5264 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚239082752501248 BDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚70206037 G@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚70 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 _step‚818 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚36812666 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 4552982528 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 122905600 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚262 trainer/global_step‚9949 NADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚59053056 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 2403768463872 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚322 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚273730425352704 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 4552982528 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 5494538240 K>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59053056 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚272125269830108 H@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚665 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚209 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚236540828090368 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚61478912 NADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚58387428 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 125829120 IBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚60 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚271326673993728 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521206 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚322 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚262 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚322 P5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5264 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654602240 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 2398760200668 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚31263687 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚8346089 G@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚60 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 2403810412032 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 2403768463872 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 831258624 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚22917528 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚36812666 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 5368709120 D=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚60 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚239082685725184 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚20643840 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚70206037 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚33393369 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 845178880 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚272125328217536 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 NGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚68 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚33393369 SFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚66776064 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚41948160 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 4517085184 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚41948160 E6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚322 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚269726509629440 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚41348068 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 5494538240 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚223 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚4293 B9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚5264 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚239082752501248 BDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚70206037 G@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚70 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 _step‚818 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚36812666 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 4552982528 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 122905600 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚262 trainer/global_step‚9949 NADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚59053056 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 2403768463872 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚322 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚273730425352704 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 4552982528 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 5494538240 K>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59053056 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚272125269830108 H@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚665 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚209 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚236540828090368 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚61478912 NADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚58387428 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 125829120 IBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚60 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚271326673993728 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521206 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚322 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚262 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚322 P8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 125829120 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚33389076 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 2398801548736 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 5368709120 EDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚31263617 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚236540807446528 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 2541878278656²C·‰2¤- _wandbruntime‚2657  _runtime‚2657øqº`s¥í &loss/total_loss‚2.2878041927350834 epoch‚24 trainer/global_step‚9949 % _timestamp‚1.7866613092391117e+09 _runtime‚2657.768944403 _step‚819 0loss/FRt1/assignment_loss‚0.6315910268696021 0loss/FRt2/assignment_loss‚0.5724934393208945 .loss/FRt1/detection_loss‚0.589594350629089 0loss/FRt2/detection_loss‚0.49412537591549777`%TPx¦ò 0loss/FRt1/assignment_loss‚0.6315910268696021 0loss/FRt2/assignment_loss‚0.5724934393208945 .loss/FRt1/detection_loss‚0.589594350629089 0loss/FRt2/detection_loss‚0.49412537591549777 &loss/total_loss‚2.2878041927350834 epoch‚24 trainer/global_step‚9949 % _timestamp‚1.7866613092391117e+09 _runtime‚2657.768944403 _step‚819³°“~•§‚(:Š É“ùÓç¸úà gpu.0.gpu‚31.0 gpu.0.memory‚4, gpu.0.memoryAllocated‚7.0313262939453125) gpu.0.memoryAllocatedBytes‚ 6039863296 gpu.0.temp‚36.0 gpu.0.powerWatts‚216.862' gpu.0.enforcedPowerLimitWatts‚500.0 gpu.0.powerPercent‚43.3724 gpu.0.smClock‚1410 gpu.0.memoryClock‚1593$ gpu.0.correctedMemoryErrors‚1955# gpu.0.uncorrectedMemoryErrors‚0s|4~¨‚(:‚ É“ùÓŒ£ ã) disk./.usagePercent‚32.71493527269349' disk.nvme0c0n1.out‚56117.5634765625 disk.dm-1.in‚25317.75 disk.drbd2165.in‚0 disk.drbd2165.out‚0$ memory_percent‚67.79301166534424# proc.memory.rssMB‚ 2354.37109375$ disk./.usageGB‚487.4761772155762 disk.sda3.in‚25317.75 disk.sda.in‚25317.75 disk.sda.out‚ 49.72265625" disk.dm-0.out‚56117.5634765625$ disk.nvme0c0n1.in‚43531.49609375 disk.sda1.in‚0 disk.sda1.out‚0 disk.sda2.out‚0" disk.nvme0n1.in‚43531.49609375% disk.nvme0n1.out‚56117.5634765625 disk.dm-0.in‚43531.49609375* proc.memory.percent‚14.369940757751465 network.recv‚ 5.074241e+06 disk.sda3.out‚ 49.72265625 disk.sda2.in‚0 disk.dm-1.out‚ 49.72265625) proc.memory.availableMB‚ 5276.79296875 cpu‚27.15512480920935 proc.cpu.threads‚17 network.sent‚ 1.3056829e+07Ê2©- _wandbruntime‚2670  _runtime‚2670ƒm‹fĪ¾ % lr-AdamW/pg1‚0.000774158835110027 % lr-AdamW/pg2‚0.000774158835110027 trainer/global_step‚9999 % _timestamp‚1.7866613217300382e+09 _runtime‚ 2670.26006836 _step‚820×g½îÉ«Ã _step‚820 % lr-AdamW/pg1‚0.000774158835110027 % lr-AdamW/pg2‚0.000774158835110027 trainer/global_step‚9999 % _timestamp‚1.7866613217300382e+09 _runtime‚ 2670.26006836´’Š¼2¬- _wandbruntime‚2670  _runtime‚2670XdI~r!­·O [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 2415166106112 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚271009613512704 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚36986763 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚41945088 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚223 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 2410132786684 @7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚5264 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 5494538240 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17104896 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚271009630552064 trainer/global_step‚9999 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚240224116975616 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚237670149357568 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚44038144 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 2410091438664 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚33550891 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚31411341 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚429 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚33555178 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 125829120 D;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚5264 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 2415166106112 PIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚75 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 5368709120 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 2415124161024 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚209 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 KBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚4876 IBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚77 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚61478912 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚272616451440640 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚41945088 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚33550891 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 2553923579904 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚322 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 4654602240 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚70541943 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4289 PCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚59049984 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚4287 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 _step‚821 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 122905600 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚275031575601664 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚272616468545536 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚273419704951368 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚36986765 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 2415124161024 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚240224052293632 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚273419763338748 PCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚58387380 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 4628521206 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 4654602240 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 2553967618048 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 4517085184 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 125829120 F?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚60 F>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚272616451440640 BDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚275031575601664 IDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4876 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 122905600 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 831258624 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚262 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚262 KDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚60 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚31411264 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚23025575 C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚4289 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17104896.Œ`ÌÂ'®¼O eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚237670149357568 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚44038144 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 2410091438664 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚33550891 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚31411341 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚429 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚33555178 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 125829120 D;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚5264 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 2415166106112 PIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚75 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 5368709120 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 2415124161024 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚209 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 KBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚4876 IBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚77 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚61478912 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚272616451440640 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚41945088 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚33550891 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 2553923579904 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚322 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 4654602240 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚70541943 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4289 PCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚59049984 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚4287 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 _step‚821 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 122905600 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚275031575601664 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚272616468545536 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚273419704951368 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚36986765 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 2415124161024 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚240224052293632 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚273419763338748 PCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚58387380 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 4628521206 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 4654602240 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 2553967618048 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 4517085184 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 125829120 F?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚60 F>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚272616451440640 BDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚275031575601664 IDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4876 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 122905600 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 831258624 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚262 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚262 KDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚60 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚31411264 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚23025575 C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚4289 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17104896 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 2415166106112 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚271009613512704 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚36986763 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚41945088 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚223 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 2410132786684 @7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚5264 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 5494538240 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17104896 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚271009630552064 trainer/global_step‚9999 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚240224116975616µæý%_2¯- _wandbruntime‚2670  _runtime‚2670ŒÝq‚Æ&°ÀM B9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚5264 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚36990316 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚272642790883328 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17104896 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 NADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚58387428 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚273446121317516 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚41948160 E6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚322 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚262 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚262 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚70544494 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚275058188061184 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚273446179704944 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 122306526 K>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59053056 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚209 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 5494538240 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚36990316 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚272642773778432 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 831258624 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚262 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚275058188061184 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚223 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 2415397177856 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚70548789 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚41348068 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 125829120 NADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚59053056 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 845178880 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚237693176086528 >5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5264 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚33558471 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚240247344500736 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚322 B8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 G@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚60 D=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚60 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 5494538240 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 125829120 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 4552982528 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚61478912 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 2410363359344 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚23027778 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚322 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 4517085184 EDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚31414262 SFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚66776064 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚36990318 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚240247411276800 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521206 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚271035816345600 PDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654602240 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚322 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 4552982528 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 2415355229696 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17104896 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚70544494 G@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚68 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚8386550 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚70548789 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 H@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚665 NGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚66 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 2415397177856M¸ïXË&±ÅM H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚4293 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚33554178 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚31414262 SFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚66776064 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚36990318 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚240247411276800 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521206 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚271035816345600 PDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654602240 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚322 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 4552982528 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 2415355229696 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17104896 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚70544494 G@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚68 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚8386550 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚70548789 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 H@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚665 NGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚66 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 2415397177856 B9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚5264 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚36990316 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚272642790883328 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17104896 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 NADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚58387428 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚273446121317516 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚41948160 E6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚322 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚262 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚262 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚70544494 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚275058188061184 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚273446179704944 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 122306526 K>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59053056 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚209 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 5494538240 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚36990316 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚272642773778432 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 831258624 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚262 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚275058188061184 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚223 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 2415397177856 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚70548789 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚41348068 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 125829120 NADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚59053056 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 845178880 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚237693176086528 >5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5264 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚33558471 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚240247344500736 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚322 B8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 G@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚60 D=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚60 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 5494538240 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 125829120 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 4552982528 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚61478912 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 2410363359344 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚23027778 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚322 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 4517085184 EDeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4289 IBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚78 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚8435898 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚20643840 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 125829120 % _timestamp‚1.7866613401018372e+09 JBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚665 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚33742472 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 831258624 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚37225021 F>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17104896 IDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚275125521902722 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚41945088 KBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚4876 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 5494538240 E8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚70963204 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 4654602240 PIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚76 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 BDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4876 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚276754284198400 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚209 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚241692123392512 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 125829120 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚322 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚262 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 122905600 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚33738185 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚272699819720704 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚262 IBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚60 F?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚60 _runtime‚2688.635871528 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚4287 M@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚59049984 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚276754343248384 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 845178880 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚262 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 5494538240 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 4654602240 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚223 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 4628521206 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚276754284198400 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚274323522650112 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 2430820598272 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 122905600 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚70967493 D;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚5264 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 2430778653184 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 4552982528 PCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚59049984 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 122306526 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 5494538240 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 5368709120 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚8435822 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚322 trainer/global_step‚10049 C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚4289 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17104896 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360W¥ãÃ'¿½O L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 4654602240 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚223 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 4628521206 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚276754284198400 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚274323522650112 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 2430820598272 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 122905600 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚70967493 D;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚5264 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 2430778653184 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 4552982528 PCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚59049984 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 122306526 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 5494538240 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 5368709120 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚8435822 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚322 trainer/global_step‚10049 C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚4289 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17104896 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚23176439 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚52426752 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 5368709120 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 125829120 _step‚826 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚37225019 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚276754343248384 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4289 IBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚78 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚8435898 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚20643840 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 125829120 % _timestamp‚1.7866613401018372e+09 JBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚665 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚33742472 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 831258624 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚37225021 F>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17104896 IDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚275125521902722 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚41945088 KBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚4876 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 5494538240 E8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚70963204 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 4654602240 PIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚76 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 BDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4876 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚276754284198400 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚209 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚241692123392512 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 125829120 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚322 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚262 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 122905600 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚33738185 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚272699819720704 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚262 IBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚60 F?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚60 _runtime‚2688.635871528 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚4287 M@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚59049984 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚276754343248384 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 845178880 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚262 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 5494538240 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0ºîYGY2À- _wandbruntime‚2688  _runtime‚2688MÌ‹K»ÁÂM HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 D6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚322 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚33745765 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚262 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17104896 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 2431051670016 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚37228574 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚33741472 I@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚4876 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚275151996656298 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚322 _step‚827 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚70974339 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚33741472 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚61478912 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 trainer/global_step‚10049 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 2431009721856 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚4293 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚31615310 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 125829120 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 5494538240 % _timestamp‚1.7866613402660615e+09 E5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5264 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 122905600 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚37228572 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚23178644 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 5368709120 B9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚5264 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚276780896657920 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚276780896657920 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚274349844987904 H@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚665 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚322 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚262 IBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚60 PDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚70974339 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚33745765 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚274349844987904 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚276780837604864 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚275151938268870 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 2570581843456 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521206 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 5494538240 D=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚60 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 2425991142058 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 5368709120 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚8436668 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚20643840 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 4654602240 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚274349827883008 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚272726005514240 NADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚59053056 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 BDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚37228572 K>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59053056 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚31615390 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚241715417521664 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚239144835678208 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 122306526 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 4552982528 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚41948160 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚52423680 EDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654602240 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚70970044 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚37228574 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 4552982528 NGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚78.œáìÍ&ÂÇM UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚41948160 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚52423680 EDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654602240 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚70970044 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚37228574 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 4552982528 NGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚78 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 D6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚322 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚33745765 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚262 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17104896 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 2431051670016 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚37228574 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚33741472 I@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚4876 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚275151996656298 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚322 _step‚827 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚70974339 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚33741472 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚61478912 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 trainer/global_step‚10049 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 2431009721856 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚4293 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚31615310 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 125829120 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 5494538240 % _timestamp‚1.7866613402660615e+09 E5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5264 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 122905600 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚37228572 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚23178644 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 5368709120 B9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚5264 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚276780896657920 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚276780896657920 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚274349844987904 H@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚665 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚322 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚262 IBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚60 PDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚70974339 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚33745765 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚274349844987904 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚276780837604864 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚275151938268870 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 2570581843456 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521206 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 5494538240 D=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚60 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 2425991142058 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 5368709120 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚8436668 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚20643840 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 4654602240 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚274349827883008 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚272726005514240 NADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚59053056 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 BDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚37228572 K>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59053056 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚31615390 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚241715417521664 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚239144835678208 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 122306526 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 4552982528»š.Þ2Ã- _wandbruntime‚2688  _runtime‚2688 YfsÄí /loss/FRt1/detection_loss‚0.5247255864342464 /loss/FRt2/detection_loss‚0.5281744778951244 epoch‚25 $ _timestamp‚1.786661340267054e+09 0loss/FRt1/assignment_loss‚0.5939048309688091 0loss/FRt2/assignment_loss‚0.5828765558817578 &loss/total_loss‚2.2296814511799377 trainer/global_step‚10049 _runtime‚2688.797301771 _step‚828¾pdxÅò /loss/FRt1/detection_loss‚0.5247255864342464 /loss/FRt2/detection_loss‚0.5281744778951244 epoch‚25 $ _timestamp‚1.786661340267054e+09 0loss/FRt1/assignment_loss‚0.5939048309688091 0loss/FRt2/assignment_loss‚0.5828765558817578 &loss/total_loss‚2.2296814511799377 trainer/global_step‚10049 _runtime‚2688.797301771 _step‚828¼Ôó;¡¡Æ‚(:– ç“ùÓ¦Õ¹á gpu.0.gpu‚64.0 gpu.0.memory‚12, gpu.0.memoryAllocated‚7.0313262939453125) gpu.0.memoryAllocatedBytes‚ 6039863296 gpu.0.temp‚35.0 gpu.0.powerWatts‚215.799' gpu.0.enforcedPowerLimitWatts‚500.0) gpu.0.powerPercent‚43.159800000000004 gpu.0.smClock‚1410 gpu.0.memoryClock‚1593$ gpu.0.correctedMemoryErrors‚1955# gpu.0.uncorrectedMemoryErrors‚0qàâb†Ç‚(:û ç“ùÓùš¥ã disk.drbd2165.out‚0 network.sent‚ 1.3193963e+07$ disk./.usageGB‚487.6021614074707$ disk.nvme0n1.out‚56270.466796875 disk.dm-1.in‚25317.75 disk.dm-1.out‚ 50.26171875 disk.drbd2165.in‚0# disk.nvme0c0n1.in‚ 43535.6640625 network.recv‚ 5.124354e+06) disk./.usagePercent‚32.72339017751931 disk.sda1.out‚0 disk.dm-0.in‚ 43535.6640625 disk.sda.out‚ 50.26171875' proc.memory.availableMB‚ 5275.765625 cpu‚27.163927681673268 disk.sda2.in‚0 disk.sda2.out‚0! disk.dm-0.out‚56270.466796875 disk.sda.in‚25317.75$ memory_percent‚67.79928207397461# proc.memory.rssMB‚ 2354.37109375* proc.memory.percent‚14.369940757751465 disk.sda1.in‚0! disk.nvme0n1.in‚ 43535.6640625 disk.sda3.in‚25317.75 disk.sda3.out‚ 50.26171875& disk.nvme0c0n1.out‚56270.466796875 proc.cpu.threads‚17)ì‚2È- _wandbruntime‚2701  _runtime‚2701[òÚWÇÉÁ trainer/global_step‚10099 $ _timestamp‚1.786661352930917e+09 _runtime‚2701.462941545 _step‚829 & lr-AdamW/pg1‚0.0007621410841303068 & lr-AdamW/pg2‚0.0007621410841303068Ný/„ÌÊÆ $ _timestamp‚1.786661352930917e+09 _runtime‚2701.462941545 _step‚829 & lr-AdamW/pg1‚0.0007621410841303068 & lr-AdamW/pg2‚0.0007621410841303068 trainer/global_step‚10099½]›˜È2Ë- _wandbruntime‚2701  _runtime‚2701ºðÀ¾'̸O NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚209 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 PCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚58387380 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 % _timestamp‚1.7866613529316103e+09 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚278081987853824 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚322 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚4287 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 4552982528 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚275639639539712 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚276446431777510 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 4654602240 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17104896 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚37402669 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚31764170 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 845178880 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚20643840 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚274009109397504 IDeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4289 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚37402671 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 5368709120 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 125829120 KDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚60 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 122905600 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚31764088 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚242856793811968 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚276446373390130 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚37402671 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚23286687 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚322 C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚4289 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚278081987853824 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 2442407364096 JBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚665 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 4628521206 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 122306526 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 _runtime‚2701.463068555 @7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚5264 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 4552982528 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 5494538240 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 125829120 _step‚830 PIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚80 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚240274156945408 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 4517085184 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 IBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚60 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17104896 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚71310245 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚223 UHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚73070592 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 2582584439808 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚262 KBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚4876 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚61478912 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 trainer/global_step‚10099 PCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚59049984 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 2442407364096 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 BDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 M@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚59049984 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚41945088 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 2442365419008 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 831258624 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚37402669 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚33907574 G>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4876 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚4287 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚33903287 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚242856720741376 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 5368709120 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 2437281031986 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚274009092358144 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 2437322380006 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚33903287 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚275639639539712 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 2582636866560 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 5494538240 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 5368709120 F?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚60 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚8477401 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 E8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚71305956 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚322 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚52426752 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚262 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 IBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚82 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚23286689 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚278082046903808 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚8477481 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚275639622434816 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚240274136301568 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚41348020 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚322 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚262 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚41945088 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 5494538240 IDeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4289 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚37402671 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 5368709120 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 125829120 KDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚60 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 122905600 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚31764088 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚242856793811968 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚276446373390130 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚37402671 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚23286687 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚322 C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚4289 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚278081987853824 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 2442407364096 JBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚665 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 4628521206 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 122306526 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 _runtime‚2701.463068555 @7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚5264 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 4552982528 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 5494538240 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 125829120 _step‚830 PIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚80 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚240274156945408 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 4517085184 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 IBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚60 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17104896 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚71310245 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚223 UHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚73070592 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 2582584439808 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚262 KBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚4876 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚61478912 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 trainer/global_step‚10099 PCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚59049984 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 2442407364096 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 BDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 M@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚59049984 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚41945088 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 2442365419008 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 831258624 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚37402669 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚33907574 G>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4876 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚4287 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚33903287 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚242856720741376 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 5368709120 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 2437281031986 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚274009092358144 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 2437322380006 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚33903287 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚275639639539712 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 2582636866560 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 5494538240¾Á32Î- _wandbruntime‚2701  _runtime‚2701R-×È&ÏÂM WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚275665944772608 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 2442638435840 SFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚68873216 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚278108600313344 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚41948160 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚33906574 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 I@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚4876 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚274035278151680 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚33910867 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521206 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚242880015930368 BDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚37406224 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 2582880485376 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚275665961877504 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 122905600 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚4293 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚61478912 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 trainer/global_step‚10099 >5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5264 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 2442596487680 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 2442596487680 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚48229376 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 K>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59053056 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654602240 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 125829120 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 122905600 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 2582832256000 _step‚831 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚37406222 E8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 % _timestamp‚1.7866613531023912e+09 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17104896 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 D=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚60 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚31767187 NGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚74 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 5494538240 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 5494538240 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚262 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚242880084803584 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 845178880 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚278108600313344 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚223 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 831258624 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 125829120 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚23288894 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚240297204318208 NADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚59053056 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚71312796 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚274035295191040 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 122306526 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚240297183674368 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 _runtime‚2701.632780031 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚33906574 P6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚322 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚276472789756278 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 2437552952666 IBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚60 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚4295 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 4552982528 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17104896 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚276472848143706 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚322 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚322 EDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚71317091 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚31767111 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚322 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚275665961877504 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚275665944772608 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚429 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚8478219 NADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚58387428 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 5368709120 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚71312796 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚33910867 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚41348068 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚262 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 4654602240_“:Í&ÐÇM G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚71317091 B9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚5264 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚8478293 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 DDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚37406224 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 2582880485376 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚275665961877504 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 122905600 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚4293 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚61478912 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 trainer/global_step‚10099 >5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5264 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 2442596487680 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 2442596487680 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚48229376 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 K>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59053056 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654602240 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 125829120 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 122905600 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 2582832256000 _step‚831 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚37406222 E8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 % _timestamp‚1.7866613531023912e+09 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17104896 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 D=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚60 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚31767187 NGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚74 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 5494538240 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 5494538240 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚262 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚242880084803584 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 845178880 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚278108600313344 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚223 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 831258624 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 125829120 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚23288894 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚240297204318208 NADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚59053056 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚71312796 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚274035295191040 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 122306526 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚240297183674368 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 _runtime‚2701.632780031 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚33906574 P6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚322 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚276472789756278 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 2437552952666 IBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚60 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚4295 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 4552982528 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17104896 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚276472848143706 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚322 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚322 EDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚71317091 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚31767111 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚322 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚275665961877504 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚275665944772608 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚429 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚8478219 NADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚58387428 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 5368709120 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚71312796 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚33910867 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚41348068 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚262 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 4654602240 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚275665944772608 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 2442638435840 SFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚68873216 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚278108600313344 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚41948160 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚33906574 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 I@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚4876 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚274035278151680 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚33910867 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521206 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚242880015930368 BDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚279409691509248 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4289 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚275318382034944 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 4517085184 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 845178880 _runtime‚2713.933874238 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚322 C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚4289 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚322 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚37580319 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 4654602240 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚34068389 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚209 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 125829120 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 2594919181312 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 125829120 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚322 KDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚60 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 4552982528 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 122905600 JBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚665 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 2448842842594 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 5368709120 _step‚834 KBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚4876 UHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚62584832 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 IBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚60 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚37580321 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 2453952184832 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚37580321 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚23396939 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚277767283264918 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚279409750559232 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚71652997 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚223 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 831258624 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚322 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17104896 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 % _timestamp‚1.7866613654006238e+09 M@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚59049984 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚34072676 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 IDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4876 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚31915277 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚429 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚41940992 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚41945088 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚276955739324416 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚241426525585408 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 122306526 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚276955756429312 IDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚31915354 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 2594877240320 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 5494538240 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 PCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚58387380 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚276955739324416 D;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚5264 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚275318399074304 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 2448884190614 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚262 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 4552982528 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚61478912 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 D>Dev%>IÞiceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 2453994129920 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 122905600 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚244021382181888 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚41348020 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 125829120 BDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 2453994129920 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 122905600 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚244021382181888 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚41348020 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 125829120 BDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚279409691509248 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4289 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚275318382034944 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 4517085184 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 845178880 _runtime‚2713.933874238 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚322 C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚4289 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚322 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚37580319 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 4654602240 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚34068389 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚209 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 125829120 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 2594919181312 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 125829120 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚322 KDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚60 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 4552982528 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 122905600 JBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚665 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 2448842842594 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 5368709120 _step‚834 KBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚4876 UHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚62584832 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 IBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚60 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚37580321 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 2453952184832 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚37580321 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚23396939 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚277767283264918 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚279409750559232 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚71652997 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚223 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 831258624 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚322 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17104896 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 % _timestamp‚1.7866613654006238e+09 M@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚59049984 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚34072676 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 IDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4876 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚31915277 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚429 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚41940992 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚41945088 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚276955739324416 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚241426525585408 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 122306526 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚276955756429312 IDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚31915354 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 2594877240320 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 5494538240 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 PCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚58387380 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚276955739324416 D;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚5264 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚275318399074304 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 2448884190614 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚262 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 4552982528Â|þ^2Ú- _wandbruntime‚2714  _runtime‚2714W“)È&ÛÂM D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 5494538240 G@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚87 DDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚23399142 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 122306526 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 5494538240 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚262 NADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚59053056 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 4552982528 H@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚665 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚244044741689344 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 2595168731136 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚34075969 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚34075969 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17104896 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚276982061662208 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 2454183253504 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚50326528 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 5494538240 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 2454183253504 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 125829120 IBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚60 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17104896 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚223 _step‚835 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚37583872 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚41948160 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 122905600 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚34071676 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 5368709120 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚279436244915712 I@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚4876 PDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚71659843 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚8519242 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚4295 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚34071676 EDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654602240 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚275344567828480 >5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5264 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚275344584867840 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 4517085184 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 4654602240 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚241449572958208 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚20643840 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521206 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚322 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚31918299 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 2595118404608 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚277793641243686 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚322 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚262 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 2454225201664 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚429 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚322 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚37583872 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚4293 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚37583874 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚262 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 2449073415206 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚322 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚209 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 2449114763274 trainer/global_step‚10149 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚71655548 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚276982078767104 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚71655548 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚37583874 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚244044670718976 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚241449552314368 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚277793699631114 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 5368709120 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 125829120 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚31918386 NGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚85 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚279436303968768 K>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59053056 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚71659843 E5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5264 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚275344584867840 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 4517085184 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 4654602240 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚241449572958208 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚20643840 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521206 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚322 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚31918299 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 2595118404608 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚277793641243686 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚322 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚262 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 2454225201664 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚429 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚322 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚37583872 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚4293 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚37583874 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚262 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 2449073415206 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚322 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚209 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 2449114763274 trainer/global_step‚10149 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚71655548 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚276982078767104 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚71655548 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚37583874 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚244044670718976 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚241449552314368 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚277793699631114 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 5368709120 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 125829120 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚31918386 NGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚85 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚279436303968768 K>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59053056 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚71659843 EDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 5494538240 G@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚87 DDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚23399142 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 122306526 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 5494538240 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚262 NADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚59053056 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 4552982528 H@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚665 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚244044741689344 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 2595168731136 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚34075969 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚34075969 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17104896 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚276982061662208 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 2454183253504 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚50326528 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 5494538240 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 2454183253504 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 125829120 IBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚60 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17104896 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚223 _step‚835 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚37583872 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚41948160 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 122905600 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚34071676 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 5368709120 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚279436244915712 I@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚4876 PDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚71659843 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚8519242 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚4295 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚34071676 EDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654602240 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚275344567828480Ç?42Ý- _wandbruntime‚2714  _runtime‚2714û€ÌuÞï 0loss/FRt2/assignment_loss‚0.5199570162972657 0loss/FRt1/detection_loss‚0.49312145543134994 /loss/FRt2/detection_loss‚0.4635242307694498 &loss/total_loss‚2.0796831945866208 epoch‚25 % _timestamp‚1.7866613655754335e+09 _runtime‚2714.105433734 _step‚836 0loss/FRt1/assignment_loss‚0.6030804920885555 trainer/global_step‚10149‘-9Kzßô 0loss/FRt2/assignment_loss‚0.5199570162972657 0loss/FRt1/detection_loss‚0.49312145543134994 /loss/FRt2/detection_loss‚0.4635242307694498 &loss/total_loss‚2.0796831945866208 epoch‚25 % _timestamp‚1.7866613655754335e+09 _runtime‚2714.105433734 _step‚836 0loss/FRt1/assignment_loss‚0.6030804920885555 trainer/global_step‚10149ÄÁ\âù¡à‚(:– ö“ùÓŽ°•á gpu.0.gpu‚62.0 gpu.0.memory‚11, gpu.0.memoryAllocated‚7.0313262939453125) gpu.0.memoryAllocatedBytes‚ 6039863296 gpu.0.temp‚35.0 gpu.0.powerWatts‚227.091' gpu.0.enforcedPowerLimitWatts‚500.0) gpu.0.powerPercent‚45.418200000000006 gpu.0.smClock‚1410 gpu.0.memoryClock‚1593$ gpu.0.correctedMemoryErrors‚1955# gpu.0.uncorrectedMemoryErrors‚0¬ˆ—1á‚(:ø ö“ùÓ¶Óæã disk.drbd2165.in‚0 disk.dm-0.out‚56277.43359375% disk.nvme0c0n1.out‚56277.43359375$ memory_percent‚67.18144416809082) proc.memory.percent‚13.75124454498291 disk.dm-1.in‚25317.75 disk.dm-0.in‚ 43535.6640625 disk.sda3.in‚25317.75# proc.memory.rssMB‚ 2253.00390625 network.recv‚ 5.145777e+06) disk./.usagePercent‚32.72342166644602$ disk./.usageGB‚487.6026306152344 disk.sda1.in‚0 disk.sda1.out‚0 disk.sda2.out‚0 disk.drbd2165.out‚0 cpu‚27.163659630264405 network.sent‚ 1.3259291e+07 disk.sda.in‚25317.75 disk.sda2.in‚0 disk.dm-1.out‚ 50.80\¬Ûó859375 disk.sda3.out‚ 50.80859375# disk.nvme0c0n1.in‚ 43535.6640625( proc.memory.availableMB‚ 5376.9921875 proc.cpu.threads‚17 disk.sda.out‚ 50.80859375! disk.nvme0n1.in‚ 43535.6640625# disk.nvme0n1.out‚56277.43359375$7û¥2â- _wandbruntime‚2726  _runtime‚2726zÀ:pÆãÀ _step‚837 % lr-AdamW/pg1‚0.000750120214004087 % lr-AdamW/pg2‚0.000750120214004087 trainer/global_step‚10199 % _timestamp‚1.7866613780650413e+09 _runtime‚2726.595051008‡ŸŽËäÅ % lr-AdamW/pg1‚0.000750120214004087 % lr-AdamW/pg2‚0.000750120214004087 trainer/global_step‚10199 % _timestamp‚1.7866613780650413e+09 _runtime‚2726.595051008 _step‚837Åßö‘¿2å- _wandbruntime‚2726  _runtime‚2726¹Æú>¼'æ¶O XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 5368709120 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 125829120 F>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚4287 G>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4876 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚278271856214016 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚8559593 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚245186048284672 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 IBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚60 IDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 @7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚5264 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 2465580895744 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚32066780 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 845178880 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚279088076364946 F?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚60 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚37757969 PCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚58387380 $ _timestamp‚1.786661378065959e+09 C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚4289 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚280737454214656 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚278271856214016 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 2460446001222 KBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚4876 UHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚64681984 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 5494538240 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚278271873318912 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚280737395164672 JBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚665 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚23507189 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚242578873581568 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 4628521206 _step‚838 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚280737454214656 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚41945088 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 4517085184 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚262 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 4552982528 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚41945088 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 4654602240 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚32066864 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 2607218741248 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 5368709120 KDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚60 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚37757971 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚41348020 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 125829120 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚322 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚322 trainer/global_step‚10199 PIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚82 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 125829120 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 2465538950656 E8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚71991460 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 5494538240 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚34237778 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚278271873318912 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 5494538240 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚322 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 2465580895744 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚262 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚37757969 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17104896 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚34233491 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚322 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚429 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚223 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 831258624 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚276627671711744 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 4654602240 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 4552982528 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚34233491 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚280737395164672 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17104896 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 2465538950656 _runtime‚ 2726.59986982 PCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚59049984 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚242578894225408 M@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚59049984 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4289 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚37757971 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚262 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚20643840 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚279088134752326 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚44038144 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 2607174703104 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 122306526 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚34237778 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 2460404653202 D;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚5264 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚23507187 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚8559675 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚209 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚276627688751104ž~—¢Á'ç»O L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 4654602240 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 4552982528 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚34233491 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚280737395164672 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17104896 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 2465538950656 _runtime‚ 2726.59986982 PCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚59049984 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚242578894225408 M@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚59049984 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4289 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚37757971 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚262 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚20643840 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚279088134752326 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚44038144 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 2607174703104 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 122306526 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚34237778 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 2460404653202 D;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚5264 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚23507187 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚8559675 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚209 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚276627688751104 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 5368709120 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 125829120 F>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚4287 G>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4876 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚278271856214016 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚8559593 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚245186048284672 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 IBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚60 IDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 @7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚5264 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 2465580895744 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚32066780 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 845178880 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚279088076364946 F?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚60 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚37757969 PCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚58387380 $ _timestamp‚1.786661378065959e+09 C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚4289 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚280737454214656 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚278271856214016 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 2460446001222 KBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚4876 UHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚64681984 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 5494538240 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚278271873318912 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚280737395164672 JBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚665 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚23507189 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚242578873581568 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 4628521206 _step‚838 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚280737454214656 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚41945088 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 4517085184 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚262 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 4552982528 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚41945088 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 4654602240 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚32066864 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 2607218741248 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 5368709120 KDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚60 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚37757971 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚41348020 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 125829120 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚322 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚322 trainer/global_step‚10199 PIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚82 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 125829120 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 2465538950656 E8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚71991460 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 5494538240 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚34237778 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚278271873318912 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 5494538240 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚322 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 2465580895744 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚262 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚37757969 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17104896 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚34233491 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚322 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚429 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚223 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 831258624 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚276627671711744Æ]ÝÁ2è- _wandbruntime‚2726  _runtime‚2726~vO²È&éÂM I@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚4876 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 2460676573882 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 G@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚60 D=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚60 NADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚58387428 DDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚32069825 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚242601941598208 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 5494538240 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚322 % _timestamp‚1.7866613782361407e+09 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚322 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚278298195656704 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 2465770019328 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚20643840 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 5494538240 EDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521206 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 4517085184 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 2460635225814 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚37761524 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚4293 H@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚665 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚262 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 2465811967488 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 122905600 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚72002595 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚223 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚279114492731094 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚41348068 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚61478912 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚242601920954368 _runtime‚2726.766452214 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚72002595 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚34241071 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 2465811967488 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚280764007624192 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 >5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5264 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 2607467078656 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚322 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 4552982528 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚4293 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚8560433 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚276653874544640 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 2465770019328 BDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654602240 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 122306526 trainer/global_step‚10199 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 122905600 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚32069908 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17104896 NADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚59053056 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 4552982528 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚23509392 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚245209408676864 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 125829120 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚262 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 125829120 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚37761522 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚71998300 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚209 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚280763948571136 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚278298178551808 G@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚83 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 5368709120 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚280764007624192 K>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59053056 P6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚322 % _timestamp‚1.7866613782361407e+09 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚322 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚278298195656704 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 2465770019328 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚20643840 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 5494538240 EDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521206 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 4517085184 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 2460635225814 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚37761524 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚4293 H@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚665 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚262 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 2465811967488 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 122905600 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚72002595 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚223 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚279114492731094 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚41348068 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚61478912 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚242601920954368 _runtime‚2726.766452214 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚72002595 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚34241071 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 2465811967488 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚280764007624192 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 >5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5264 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 2607467078656 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚322 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 4552982528 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚4293 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚8560433 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚276653874544640 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 2465770019328 BDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654602240 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 122306526 trainer/global_step‚10199 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 122905600 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚32069908 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17104896 NADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚59053056 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 4552982528 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚23509392 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚245209408676864 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 125829120 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚262 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 125829120 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚37761522 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚71998300 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚209 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚280763948571136 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚278298178551808 G@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚83 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 5368709120 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚280764007624192 K>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59053056 PDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚32069825 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚242601941598208Ç)uD‹2ë- _wandbruntime‚2726  _runtime‚2726@„.®oìé #loss/total_loss‚2.4212804363156 epoch‚25 trainer/global_step‚10199 % _timestamp‚1.7866613782372584e+09 _runtime‚2726.767111399 0loss/FRt1/assignment_loss‚0.7359859197354843 -loss/FRt1/detection_loss‚0.52295484074022 /loss/FRt2/detection_loss‚0.4974729224679958 _step‚840 0loss/FRt2/assignment_loss‚0.6648667533719002ÂR·tíî 0loss/FRt1/assignment_loss‚0.7359859197354843 -loss/FRt1/detection_loss‚0.52295484074022 /loss/FRt2/detection_loss‚0.4974729224679958 _step‚840 0loss/FRt2/assignment_loss‚0.6648667533719002 #loss/total_loss‚2.4212804363156 epoch‚25 trainer/global_step‚10199 % _timestamp‚1.7866613782372584e+09 _runtime‚2726.767111399ÈÕ³§ì–î‚(:‹ …”ùÓçôà gpu.0.gpu‚64.0 gpu.0.memory‚12, gpu.0.memoryAllocated‚7.0313262939453125) gpu.0.memoryAllocatedBytes‚ 6039863296 gpu.0.temp‚36.0 gpu.0.powerWatts‚196.386' gpu.0.enforcedPowerLimitWatts‚500.0 gpu.0.powerPercent‚39.2772 gpu.0.smClock‚1410 gpu.0.memoryClock‚1593$ gpu.0.correctedMemoryErrors‚1955# gpu.0.uncorrectedMemoryErrors‚04<ÙéŒï‚(: …”ùÓàÙþâ disk.sda2.in‚0 disk.drbd2165.out‚0$ memory_percent‚67.18065738677979* proc.memory.percent‚13.751363754272461( disk.nvme0c0n1.out‚56321.62158203125 disk.sda2.out‚0 disk.dm-1.in‚25317.75# disk.dm-0.out‚56321.62158203125 proc.cpu.threads‚17 network.sent‚ 1.3346568e+07 network.recv‚ 5.169641e+06$ disk./.usageGB‚487.6026725769043 disk.sda3.in‚25317.75 disk.sda.in‚25317.75 disk.sda1.in‚0& disk.nvme0n1.out‚56321.62158203125 disk.dm-0.in‚ 43535.6640625 disk.sda3.out‚ 51.26953125 disk.sda.out‚ 51.26953125! disk.nvme0n1.in‚ 43535.6640625 disk.dm-1.out‚ 51.26953125 disk.drbd2165.in‚0) proc.memory.availableMB‚ 5377.12109375" proc.memory.rssMB‚ 2253.0234375 cpu‚27.163150302669163( disk./.usagePercent‚32.7234244825289# disk.nvme0c0n1.in‚ 43535.6640625 disk.sda1.out‚0Qld’2ð- _wandbruntime‚2739  _runtime‚2739¯zÂÇñÁ & lr-AdamW/pg1‚0.0007441095743334668 & lr-AdamW/pg2‚0.0007441095743334668 trainer/global_step‚10249 $ _timestamp‚1.786661390969392e+09 _runtime‚2739.499398578 _step‚841šZ‚ßÌòÆ & lr-AdamW/pg1‚0.0007441095743334668 & lr-AdamW/pg2‚0.0007441095743334668 trainer/global_step‚10249 $ _timestamp‚1.786661390969392e+09 _runtime‚2739.499398578 _step‚841É%ýØ2ó- _wandbruntime‚2739  _runtime‚2739däâ_¾'ô¸O \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚277936961388544 F>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 D;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚5264 UHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚58390528 G>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4876 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚243731242221568 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 4517085184 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 5494538240 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 2477167661568 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚246350720201216 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 2619515726336 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚277936978427904 C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚4289 E8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚72334212 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚279587990208512 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚32218144 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚20643840 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 2472007811830 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚279587990208512 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚41945088 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚209 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚243731262865408 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 4552982528 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 122905600 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 2477125716480 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚322 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17104896 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚429 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚280408927852354 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚34402880 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 5494538240 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 125829120 F?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚60 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚37935619 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 122306526 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚322 _runtime‚2739.503871106 @7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚5264 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 IBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚60 KBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚4876 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚8600705 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚322 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚4287 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚4287 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚322 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚72338501 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 2619477979648 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚280408986239734 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 5494538240 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 5368709120 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 5368709120 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚262 KDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚60 _step‚842 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚34398593 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚282065157870080 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 4654602240 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 PCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚58387380 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚37935621 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 845178880 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 trainer/global_step‚10249 % _timestamp‚1.7866613909702961e+09 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚282065098820096 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 2477167661568 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚23617437 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚8600645 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚32218082 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚37746688 BDeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4289 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 2471966463810 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 2477125716480 JBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚665 IBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚62 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 831258624Ü„>Ã'õ½O `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚279587990208512 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚41945088 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚209 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚243731262865408 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 4552982528 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 122905600 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 2477125716480 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚322 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17104896 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚429 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚280408927852354 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚34402880 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 5494538240 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 125829120 F?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚60 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚37935619 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 122306526 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚322 _runtime‚2739.503871106 @7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚5264 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 IBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚60 KBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚4876 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚8600705 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚322 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚4287 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚4287 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚322 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚72338501 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 2619477979648 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚280408986239734 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 5494538240 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 5368709120 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 5368709120 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚262 KDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚60 _step‚842 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚34398593 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚282065157870080 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 4654602240 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 PCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚58387380 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚37935621 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 845178880 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 trainer/global_step‚10249 % _timestamp‚1.7866613909702961e+09 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚282065098820096 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 2477167661568 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚23617437 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚8600645 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚32218082 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚37746688 BDeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4289 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 2471966463810 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 2477125716480 JBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚665 IBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚62 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 831258624 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚277936961388544 F>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 D;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚5264 UHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚58390528 G>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4876 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚243731242221568 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 4517085184 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 5494538240 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 2477167661568 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚246350720201216 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 2619515726336 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚277936978427904 C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚4289 E8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚72334212 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚279587990208512 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚32218144 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚20643840 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 2472007811830ʹ‚V2ö- _wandbruntime‚2739  _runtime‚2739h|jÁK÷ÂM QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 845178880 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚243754289594368 % _timestamp‚1.7866613911356783e+09 E6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚322 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚429 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚223 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚20643840 B5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5264 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚34406173 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚34401880 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 4517085184 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 2472238384490 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚262 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚8601563 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚262 trainer/global_step‚10249 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚72341052 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 4552982528 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 2477398733312 B9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚5264 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚37939172 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 5368709120 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521206 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 122905600 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚37939174 I@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚4876 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚246374074465792 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚322 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚277963147182080 G@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚76 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚280435344218502 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚41948160 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 _step‚843 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚279614312546304 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚279614295441408 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚23619642 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚4293 E8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚280435402605930 H@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚665 NADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚58387428 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 122306526 PDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚8601489 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 IBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚60 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚37939172 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚282091711279616 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚34401880 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 4654602240 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654602240 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚279614312546304 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚279614295441408 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚32221131 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 5494538240 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17104896 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 2477398733312 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚72345347 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚246374007689728 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 5494538240 NGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚74 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 2619718095360 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚72345347 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚282091711279616 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 5368709120 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 125829120 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚37939174 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚41948160 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚46132224 D=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚60 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 2477356785152 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚243754310238208 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 2619764227584 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚277963164221440 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 125829120 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚41348068 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 5494538240 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚4295 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 122905600 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚32221207 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚23619644 SFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚66776064 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 2472197036422 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚4293 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 4552982528 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚209 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 831258624 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚61478912 DDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59053056 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚72341052 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 125829120 _runtime‚2739.666012052>”X›Í&øÇM MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚280435402605930 H@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚665 NADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚58387428 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 122306526 PDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚8601489 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 IBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚60 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚37939172 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚282091711279616 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚34401880 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 4654602240 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654602240 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚279614312546304 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚279614295441408 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚32221131 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 5494538240 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17104896 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 2477398733312 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚72345347 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚246374007689728 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 5494538240 NGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚74 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 2619718095360 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚72345347 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚282091711279616 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 5368709120 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 125829120 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚37939174 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚41948160 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚46132224 D=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚60 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 2477356785152 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚243754310238208 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 2619764227584 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚277963164221440 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 125829120 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚41348068 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 5494538240 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚4295 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 122905600 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚32221207 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚23619644 SFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚66776064 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 2472197036422 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚4293 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 4552982528 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚209 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 831258624 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚61478912 DDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59053056 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚72341052 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 125829120 _runtime‚2739.666012052 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 845178880 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚243754289594368 % _timestamp‚1.7866613911356783e+09 E6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚322 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚429 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚223 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚20643840 B5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5264 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚34406173 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚34401880 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 4517085184 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 2472238384490 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚262 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚8601563 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚262 trainer/global_step‚10249 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚72341052 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 4552982528 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 2477398733312 B9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚5264 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚37939172 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 5368709120 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521206 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 122905600 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚37939174 I@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚4876 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚246374074465792 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚322 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚277963147182080 G@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚76 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚280435344218502 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚41948160 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 _step‚843 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚279614312546304 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚279614295441408 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚23619642 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚4293 EDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚283392802475520 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 122905600 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚279246268104704 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 4552982528 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 4654602240 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚32369392 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 831258624 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 5368709120 C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚4289 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚34567982 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 4628521206 JBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚665 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 G>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4876 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 4654602240 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚244883631505408 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚280904107098112 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 2488712482304 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4289 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚23727689 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚244883610861568 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 2631772419072 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚322 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 5494538240 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚322 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚322 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚262 IBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚73 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚41348020 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚32369319 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚209 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚283392861525504 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚38113269 % _timestamp‚1.7866614034173856e+09 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚34563695 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚280904107098112 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚223 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚8641703 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚20643840 BDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚4287 _step‚846 PCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚59049984 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚34563695 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚46135296 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 5494538240 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 125829120 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚4287 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 4552982528 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚247515383280640 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 IDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 KDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚60 M@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚59049984 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17104896 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚41945088 D;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚5264 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0j ²fhƒ½O YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚283392861525504 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚38113269 % _timestamp‚1.7866614034173856e+09 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚34563695 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚280904107098112 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚223 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚8641703 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚20643840 BDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚4287 _step‚846 PCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚59049984 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚34563695 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚46135296 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 5494538240 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 125829120 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚4287 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 4552982528 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚247515383280640 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 IDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 KDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚60 M@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚59049984 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17104896 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚41945088 D;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚5264 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚281729779339762 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚429 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 2483528274418 E8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚72676964 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚280904089993216 UHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚66779136 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚38113269 KBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚4876 PCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚58387380 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚262 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚262 IBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚60 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚34567982 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 122905600 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚247515450059776 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚72681253 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 4517085184 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚281729837727142 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚283392802475520 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 122905600 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚279246268104704 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 4552982528 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 4654602240 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚32369392 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 831258624 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 5368709120 C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚4289 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚34567982 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 4628521206 JBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚665 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 G>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4876 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 4654602240 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚244883631505408 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚280904107098112 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 2488712482304 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4289 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚23727689 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚244883610861568 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 2631772419072 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚322 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 5494538240 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚322 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚322 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚262 IBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚73 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚41348020 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚32369319 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚209 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0ÎCÏ<‡2„- _wandbruntime‚2752  _runtime‚2752î@0‰È&…ÂM HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚281756195705910 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚279272436858880 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 5368709120 _step‚847 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚4295 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚280930429435904 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚244906678878208 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 122306526 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 4552982528 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚4293 NGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚83 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚61478912 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 5368709120 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 2632066531328 BDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚72688099 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚247538674439168 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚20643840 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚322 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚38116822 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚34571275 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚280930429435904 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚34571275 SFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚70970368 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚23729892 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 G@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚60 NADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚58387428 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 4517085184 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚223 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 2488985499136 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚244906658234368 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚280930412331008 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚41948160 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚23729894 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚283419414935040 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚34566982 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚32372358 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚8642466 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚262 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 H@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚665 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 DDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654602240 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚41348068 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 4654602240 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚280930412331008 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚283419355881984 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚279272453898240 K>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59053056 EDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 IBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚60 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚41948160 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚429 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚209 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 5494538240 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 5368709120 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚322 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚322 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚262 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 122905600 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 4552982528 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 2488985499136 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 125829120 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚34566982 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚247538745409536 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚283419414935040 >5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5264 B9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚5264 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚322 PDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521206 trainer/global_step‚10299 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 122905600 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚262 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚38116822î}HòÍ&†ÇM E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 G@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚60 NADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚58387428 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 4517085184 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚223 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 2488985499136 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚244906658234368 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚280930412331008 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚41948160 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚23729894 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚283419414935040 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚34566982 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚32372358 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚8642466 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚262 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 H@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚665 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 DDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654602240 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚41348068 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 4654602240 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚280930412331008 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚283419355881984 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚279272453898240 K>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59053056 EDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 IBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚60 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚41948160 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚429 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚209 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 5494538240 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 5368709120 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚322 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚322 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚262 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 122905600 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 4552982528 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 2488985499136 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 125829120 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚34566982 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚247538745409536 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚283419414935040 >5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5264 B9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚5264 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚322 PDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521206 trainer/global_step‚10299 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 122905600 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚262 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚38116822 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚281756195705910 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚279272436858880 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 5368709120 _step‚847 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚4295 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚280930429435904 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚244906678878208 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 122306526 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 4552982528 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚4293 NGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚83 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚61478912 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 5368709120 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 2632066531328 BDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚72688099 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚247538674439168 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚20643840 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚322 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚38116822 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚34571275 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚280930429435904 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚34571275 SFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚70970368 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚23729892Ï7¸¥Í2‡- _wandbruntime‚2752  _runtime‚2752ÊwØ©tˆî 0loss/FRt1/assignment_loss‚0.6448296061629191 0loss/FRt2/assignment_loss‚0.5031520764606897 /loss/FRt1/detection_loss‚0.5107358111632241 &loss/total_loss‚2.1376644486461345 epoch‚25 % _timestamp‚1.7866614035809734e+09 /loss/FRt2/detection_loss‚0.4789469548593019 trainer/global_step‚10299 _runtime‚2752.110791486 _step‚848ˆñµÉy‰ó 0loss/FRt1/assignment_loss‚0.6448296061629191 0loss/FRt2/assignment_loss‚0.5031520764606897 /loss/FRt1/detection_loss‚0.5107358111632241 &loss/total_loss‚2.1376644486461345 epoch‚25 % _timestamp‚1.7866614035809734e+09 /loss/FRt2/detection_loss‚0.4789469548593019 trainer/global_step‚10299 _runtime‚2752.110791486 _step‚848Ð.iÂÔ•Š‚(:Š £”ùÓ°™›á gpu.0.gpu‚60.0 gpu.0.memory‚11, gpu.0.memoryAllocated‚7.0313262939453125) gpu.0.memoryAllocatedBytes‚ 6039863296 gpu.0.temp‚36.0 gpu.0.powerWatts‚217.925' gpu.0.enforcedPowerLimitWatts‚500.0 gpu.0.powerPercent‚43.585 gpu.0.smClock‚1410 gpu.0.memoryClock‚1593$ gpu.0.correctedMemoryErrors‚1955# gpu.0.uncorrectedMemoryErrors‚0±[Û—‹‚(:Œ £”ùÓ¡±Èã disk.sda.in‚25988.94921875 disk.sda.out‚ 51.66015625 disk.sda1.out‚0* proc.memory.percent‚13.751387596130371 cpu‚27.162640072342164 network.sent‚ 1.3467163e+07" disk.nvme0n1.in‚43559.59765625 disk.dm-0.in‚43559.59765625$ memory_percent‚67.17987060546875# proc.memory.availableMB‚5377.25 proc.cpu.threads‚17) disk./.usagePercent‚32.72238406790975 disk.sda2.out‚0$ disk.nvme0n1.out‚56373.904296875 disk.dm-1.in‚25988.94921875 disk.sda1.in‚0# proc.memory.rssMB‚ 2253.02734375! disk.dm-0.out‚56373.904296875 network.recv‚ 5.202733e+06 disk.sda2.in‚0 disk.dm-1.out‚ 51.66015625 disk.drbd2165.in‚0 disk.drbd2165.out‚0 disk.sda3.in‚25988.94921875$ disk.nvme0c0n1.in‚43559.59765625$ disk./.usageGB‚487.5871696472168 disk.sda3.out‚ 51.66015625& disk.nvme0c0n1.out‚56373.904296875K¨2Œ- _wandbruntime‚2764  _runtime‚2764·O¿È & lr-AdamW/pg1‚0.0007320898160131806 & lr-AdamW/pg2‚0.0007320898160131806 trainer/global_step‚10349 % _timestamp‚1.7866614161808481e+09 _runtime‚2764.710864671 _step‚8499ÕzÍŽÇ trainer/global_step‚10349 % _timestamp‚1.7866614161808481e+09 _runtime‚2764.710864671 _step‚849 & lr-AdamW/pg1‚0.0007320898160131806 & lr-AdamW/pg2‚0.0007320898160131806Ñé<1Q2- _wandbruntime‚2764  _runtime‚2764Œñ8¸O G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4289 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚38290921 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚34728797 KBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚4876 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 2495090085026 IBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚60 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚8682964 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚283050630827170 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 122306526 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚262 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 4552982528 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚41945088 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚248680026822144 =6DeviceStatsMo¬,dφ"nitor.on_train_batch_start/max_split_size‚-1 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 125829120 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚34733084 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚248680095698432 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚61478912 PCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚58387380 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚322 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚322 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚282220206882816 IDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 5494538240 E8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚73019716 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚4287 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 4628521206 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚280555540742144 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 5368709120 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 125829120 C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚4289 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚223 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 2644095553024 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚284720506130944 JBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚665 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 831258624 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 4517085184 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 5368709120 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17104896 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚41945088 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 4654602240 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚246036000145408 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚322 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚262 F?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚60 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚284720565180928 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚38290919 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚38290919 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚284720565180928 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 2500299248128 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚23837939 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚280555557781504 % _timestamp‚1.7866614161817575e+09 PCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚59049984 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 trainer/global_step‚10349 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚38290921 D;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚5264 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 122905600 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 2500299248128 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚209 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚41348020 F>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚34728797 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚73024005 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 122905600 UHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚68876288 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 5494538240 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 _step‚850 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17104896 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚8682888 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚48232448 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚283050689214550 PIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚76 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 125829120 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚282220223987712 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚282220206882816 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚20643840 _runtime‚2764.714863214 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚4287 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚262 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 4654602240 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 2500341193216 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚32520903 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 845178880 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 IDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚284720506130944 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 2500341193216 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚322 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚282220223987712 BDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4876 IBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚78 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 5368709120‰®>Ã'‘½O JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚262 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 4552982528 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚41945088 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚248680026822144 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 125829120 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚34733084 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚248680095698432 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚61478912 PCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚58387380 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚322 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚322 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚282220206882816 IDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 5494538240 E8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚73019716 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚4287 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 4628521206 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚280555540742144 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 5368709120 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 125829120 C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚4289 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚223 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 2644095553024 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚284720506130944 JBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚665 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 831258624 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 4517085184 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 5368709120 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17104896 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚41945088 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 4654602240 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚246036000145408 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚322 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚262 F?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚60 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚284720565180928 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚38290919 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚38290919 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚284720565180928 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 2500299248128 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚23837939 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚280555557781504 % _timestamp‚1.7866614161817575e+09 PCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚59049984 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 trainer/global_step‚10349 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚38290921 D;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚5264 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 122905600 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 2500299248128 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚209 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚41348020 F>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚34728797 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚73024005 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 122905600 UHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚68876288 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 5494538240 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 _step‚850 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17104896 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚8682888 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚48232448 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚283050689214550 PIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚76 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 125829120 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚282220223987712 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚282220206882816 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚20643840 _runtime‚2764.714863214 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚4287 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚262 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 4654602240 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 2500341193216 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚32520903 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 845178880 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 IDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚284720506130944 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 2500341193216 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚322 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚282220223987712 BDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4876 IBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚78 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 5368709120 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4289 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚38290921 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚34728797 KBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚4876 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 2495090085026 IBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚60 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚8682964 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚283050630827170 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 122306526 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0Ò·2’- _wandbruntime‚2764  _runtime‚2764ȲÆgÈ&“ÂM ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚282246529220608 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚61478912 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚41948160 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚41948160 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚32523840 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚248703316932096 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 5368709120 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚38294474 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚322 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚283077105580746 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 G@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚60 NADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚59053056 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 845178880 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚246059026874368 NADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚58387428 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521206 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚322 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚41348068 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚282246546325504 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 2500572264960 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚73030851 _runtime‚2764.877494914 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚34732084 I@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚4876 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17104896 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚32523924 G@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚84 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17104896 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 831258624 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 2644290057728 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 % _timestamp‚1.7866614163471837e+09 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚4293 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 5494538240 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚73030851 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 4552982528 IBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚60 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚4295 D5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5264 H@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚665 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚23840142 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 5368709120 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654602240 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚262 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚34736377 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚282246529220608 B8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 5494538240 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 125829120 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 2500530316800 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚429 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚223 _step‚851 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚73026556 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 122905600 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚4293 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚284747059537408 PDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 K>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59053056 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚209 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚280581743575040 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚322 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚322 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚38294474 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚38294472 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚73026556 D=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚60sö&1š”ÇM XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 831258624 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 2644290057728 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 % _timestamp‚1.7866614163471837e+09 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚4293 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 5494538240 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚73030851 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 4552982528 IBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚60 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚4295 D5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5264 H@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚665 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚23840142 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 5368709120 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654602240 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚262 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚34736377 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚282246529220608 B8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 5494538240 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 125829120 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 2500530316800 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚429 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚223 _step‚851 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚73026556 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 122905600 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚4293 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚284747059537408 PDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 K>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59053056 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚209 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚280581743575040 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚322 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚322 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚38294474 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚38294472 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚73026556 D=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚60 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚282246529220608 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚61478912 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚41948160 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚41948160 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚32523840 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚248703316932096 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 5368709120 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚38294474 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚322 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚283077105580746 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 G@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚60 NADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚59053056 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 845178880 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚246059026874368 NADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚58387428 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521206 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚322 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚41348068 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚282246546325504 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 2500572264960 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚73030851 _runtime‚2764.877494914 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚34732084 I@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚4876 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17104896 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚32523924 G@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚84 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17104896ÓíÕ¨­2•- _wandbruntime‚2764  _runtime‚2764“€s–í %loss/total_loss‚2.283818027931733 epoch‚25 % _timestamp‚1.7866614163483613e+09 _runtime‚2764.878225645 _step‚852 0loss/FRt2/assignment_loss‚0.5995597049598625 trainer/global_step‚10349 0loss/FRt1/assignment_loss‚0.6375847564549042 /loss/FRt1/detection_loss‚0.5427719126903376 /loss/FRt2/detection_loss‚0.5039016538266287o²>x—ò 0loss/FRt1/assignment_loss‚0.6375847564549042 /loss/FRt1/detection_loss‚0.5427719126903376 /loss/FRt2/detection_loss‚0.5039016538266287 %loss/total_loss‚2.283818027931733 epoch‚25 % _timestamp‚1.7866614163483613e+09 _runtime‚2764.878225645 _step‚852 0loss/FRt2/assignment_loss‚0.5995597049598625 trainer/global_step‚10349Ô9¦h –˜‚(:‹ ²”ùÓé¢öà gpu.0.gpu‚64.0 gpu.0.memory‚12, gpu.0.memoryAllocated‚7.0313262939453125) gpu.0.memoryAllocatedBytes‚ 6039863296 gpu.0.temp‚35.0 gpu.0.powerWatts‚223.327' gpu.0.enforcedPowerLimitWatts‚500.0 gpu.0.powerPercent‚44.6654 gpu.0.smClock‚1410 gpu.0.memoryClock‚1593$ gpu.0.correctedMemoryErrors‚1955# gpu.0.uncorrectedMemoryErrors‚0œ³2¦¡™‚(:– ²”ùÓ‰™ôã proc.cpu.threads‚17 network.sent‚ 1.3532128e+07 disk.sda3.in‚25988.94921875( disk.nvme0c0n1.out‚56381.61962890625 disk.sda1.out‚0 network.recv‚ 5.224006e+06 disk.dm-1.in‚25988.94921875 disk.drbd2165.in‚0 disk.dm-0.in‚43559.59765625# disk.dm-0.out‚56381.61962890625$ disk.nvme0c0n1.in‚43559.59765625) proc.memory.availableMB‚ 5377.01953125* proc.memory.percent‚13.751387596130371) disk./.usagePercent‚32.72343830693575% disk./.usageGB‚487.60287857055664" disk.nvme0n1.in‚43559.59765625& disk.nvme0n1.out‚56381.61962890625 disk.dm-1.out‚ 51.8203125 disk.sda3.out‚ 51.8203125 disk.sda1.in‚0# proc.memory.rssMB‚ 2253.02734375 disk.sda2.in‚0 disk.sda2.out‚0 disk.drbd2165.out‚0 disk.sda.in‚25988.94921875 disk.sda.out‚ 51.8203125$ memory_percent‚67.18127727508545 cpu‚27.162477259658054§×D¦2š- _wandbruntime‚2777  _runtime‚2777C{5ț & lr-AdamW/pg1‚0.0007260814693701315 & lr-AdamW/pg2‚0.0007260814693701315 trainer/global_step‚10399 % _timestamp‚1.7866614286208599e+09 _runtime‚2777.152949202 _step‚853¼lÍœÇ _runtime‚2777.152949202 _step‚853 & lr-AdamW/pg1‚0.0007260814693701315 & lr-AdamW/pg2‚0.0007260814693701315 trainer/global_step‚10399 % _timestamp‚1.7866614286208599e+09Õ\.¼2- _wandbruntime‚2777  _runtime‚2777ÝêDL½'ž·O WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚41348020 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚262 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚283536340877312 JBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚665 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚322 IBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚77 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚286048209786368 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚32671914 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚46135296 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚322 F>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 D;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚5264 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 _runtime‚2777.153081192 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 2511886013952 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 2511927959040 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚34893899 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 4654602240 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17104896 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚20643840 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 122306526 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚4287 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 F?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚60 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚38468571 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚38468569 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚23948189 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 2656393097216 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 5368709120 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 trainer/global_step‚10399 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚247188348141568 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 2506693243654 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 4552982528 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚286048268836352 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4289 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚4287 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚284371540701958 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚34893899 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 4628521206 IBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚60 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 $ _timestamp‚1.786661428621548e+09 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚41945088 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 4654602240 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚429 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚41945088 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚32671837 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚223 PIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚75 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚249844695103488 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 831258624 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 125829120 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 125829120 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚322 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚38468571 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚286048209786368 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚283536323772416 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚281864847458304 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚38468569 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚249844761882624 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 2656346961920 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 5494538240 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚262 _step‚854 M@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚59049984 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 5368709120 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 4552982528 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 122905600 PCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚59049984 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚286048268836352 IDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4876 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17104896 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚34898186 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚281864830418944 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚284371482314578 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 5368709120 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 845178880 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 4517085184 IDeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4289 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚4287 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚284371540701958 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚34893899 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 4628521206 IBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚60 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 $ _timestamp‚1.786661428621548e+09 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚41945088 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 4654602240 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚429 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚41945088 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚32671837 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚223 PIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚75 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚249844695103488 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 831258624 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 125829120 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 125829120 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚322 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚38468571 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚286048209786368 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚283536323772416 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚281864847458304 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚38468569 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚249844761882624 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 2656346961920 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 5494538240 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚262 _step‚854 M@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚59049984 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 5368709120 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 4552982528 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 122905600 PCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚59049984 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚286048268836352 IDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4876 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17104896 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚34898186 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚281864830418944 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚284371482314578 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 5368709120 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 845178880 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 4517085184 IDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚286048209786368 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚32671914 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚46135296 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚322 F>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 D;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚5264 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 _runtime‚2777.153081192 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 2511886013952 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 2511927959040 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚34893899 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 4654602240 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17104896 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚20643840 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 122306526 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚4287 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 F?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚60 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚38468571 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚38468569 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚23948189 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 2656393097216 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 5368709120 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 trainer/global_step‚10399 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚247188348141568 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 2506693243654 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 4552982528 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0Ö€Ó°2 - _wandbruntime‚2777  _runtime‚2777±z Ém ¡ÂM NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚38472122 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚4293 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚281891016212480 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 5494538240 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 125829120 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚322 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚262 K>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59053056 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚283562663215104 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 % _timestamp‚1.7866614287939415e+09 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚34901479 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚73369308 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 2512159030784 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚32674943 NADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚58387428 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚281891033251840 D6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚322 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 2506923816314 EDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 I@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚4876 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚247211395514368 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 831258624 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 _step‚855 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚429 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚249868058805248 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚247211416158208 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17104896 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚286074822245888 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚249867985737728 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 2656590223360 IBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚60 >5DeviceStatsMonitor.on_train_batch_end/active.™ÃY[all.peak‚5264 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚283562646110208 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚8724478 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚4295 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚286074763192832 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 2512117082624 B9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚5264 SFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚73067520 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚20643840 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 2656642647040 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 4517085184 D=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚60 EDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚73373603 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚61478912 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 BDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654602240 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚23950392 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 5368709120 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 125829120 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚34897186 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 4654602240 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 4552982528 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚41948160 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚262 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 2512117082624 G@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚73 NGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚71 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521206 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 5494538240 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚262 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚38472124 H@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚665 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚34897186 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚23950394 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚322 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 122905600 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚32674870 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚322 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚284397898680726 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚286074822245888 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚4293 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 4552982528 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 NADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚59053056 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚8724549 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 5368709120 _runtime‚2777.324350194 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚73373603 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚73369308 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚209 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚41948160 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚41348068 P6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚322 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 2506923816314 EDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 I@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚4876 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚247211395514368 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 831258624 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 _step‚855 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚429 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚249868058805248 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚247211416158208 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17104896 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚286074822245888 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚249867985737728 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 2656590223360 IBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚60 >5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5264 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚283562646110208 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚8724478 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚4295 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚286074763192832 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 2512117082624 B9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚5264 SFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚73067520 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚20643840 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 2656642647040 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 4517085184 D=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚60 EDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚73373603 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚61478912 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 BDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654602240 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚23950392 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 5368709120 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 125829120 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚34897186 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 4654602240 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 4552982528 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚41948160 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚262 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 2512117082624 G@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚73 NGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚71 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521206 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 5494538240 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚262 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚38472124 H@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚665 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚34897186 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚23950394 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚322 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 122905600 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚32674870 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚322 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚284397898680726 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚286074822245888 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚4293 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 4552982528 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 NADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚59053056 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚8724549 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 5368709120 _runtime‚2777.324350194 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚73373603 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚73369308 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚209 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚41948160 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚41348068 PDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59053056 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚283562663215104 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 % _timestamp‚1.7866614287939415e+09 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚34901479 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚73369308 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 2512159030784 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚32674943 NADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚58387428 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚281891033251840 DDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4876 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 2527540503552 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 5368709120 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 5368709120 M@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚59049984 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 2527540503552 D;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚5264 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚286077357653307 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 2522279678343 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17104896 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚322 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚322 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚287770977430528 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 4552982528 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4289 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚8774051 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 PCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚58387380 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 122306526 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚41945088 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚209 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚20643840 F?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚60 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚286077299265927 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 125829120 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚24099051 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚4287 PCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚59049984 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 831258624 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚50329600 BDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚287770918380544 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 122905600 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚38706827 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚61478912 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚32873104 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚429 _step‚859 IDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚35085475 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 4552982528 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚35081188 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚322 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 2527582448640 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17104896 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚41945088 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚73792302 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 IBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚67 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚283555036626944 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 5368709120 % _timestamp‚1.7866614469182653e+09 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚38706825 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚248639987089408 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 125829120 IDeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4289 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚8774051 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 PCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚58387380 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 122306526 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚41945088 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚209 \ODeviceSQæà>$tatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚20643840 F?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚60 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚286077299265927 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 125829120 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚24099051 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚4287 PCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚59049984 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 831258624 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚50329600 BDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚287770918380544 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 122905600 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚38706827 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚61478912 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚32873104 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚429 _step‚859 IDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚35085475 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 4552982528 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚35081188 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚322 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 2527582448640 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17104896 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚41945088 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚73792302 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 IBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚67 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚283555036626944 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 5368709120 % _timestamp‚1.7866614469182653e+09 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚38706825 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚248639987089408 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 125829120 IDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4876 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 2527540503552 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 5368709120 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 5368709120 M@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚59049984 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 2527540503552 D;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚5264 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚286077357653307 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 2522279678343 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17104896 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0ÛèE¸Î2±- _wandbruntime‚2795  _runtime‚2795:´õÎÈ&²ÂM ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚35084475 E6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚322 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚41948160 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚24101256 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚322 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521206 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 IBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚60 _step‚860 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚24101258 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚8774816 SFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚70970368 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 4654602240 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 831258624 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 122306526 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚285269717319680 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚35084475 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚73794853 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚38710380 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚248663055106048 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚283581205381120 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚38710378 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚4293 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚248663034462208 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 2673015375360 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚286103715632075 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚38710380 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚35088768 BDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚73799148 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚61478912 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 5494538240 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚73794853 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚322 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚322 NADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚58387428 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 125829120 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚262 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 125829120 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚32876151 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17104896 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 2527771572224 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚50326528 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚41348068 >5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5264 NADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚59053056 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚251335999511040 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚20643840 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 5368709120 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚287797530840064 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚223 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 2522551599023 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 5494538240 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654602240 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚286103774019503 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 2527813520384 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚285269700214784 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚8774893 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚251336070481408 DDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚32876072 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚262 _runtime‚2795.627245094 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 EDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59053056#ˆ«Í&³ÇM SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 122306526 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚285269717319680 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚35084475 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚73794853 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚38710380 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚248663055106048 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚283581205381120 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚38710378 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚4293 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚248663034462208 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 2673015375360 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚286103715632075 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚38710380 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚35088768 BDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚73799148 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚61478912 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 5494538240 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚73794853 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚322 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚322 NADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚58387428 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 125829120 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚262 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 125829120 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚32876151 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17104896 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 2527771572224 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚50326528 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚41348068 >5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5264 NADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚59053056 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚251335999511040 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚20643840 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 5368709120 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚287797530840064 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚223 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 2522551599023 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 5494538240 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654602240 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚286103774019503 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 2527813520384 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚285269700214784 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚8774893 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚251336070481408 DDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚32876072 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚262 _runtime‚2795.627245094 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 EDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59053056 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚35084475 E6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚322 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚41948160 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚24101256 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚322 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521206 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 IBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚60 _step‚860 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚24101258 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚8774816 SFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚70970368 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 4654602240 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 831258624 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0ÜtÜ2´- _wandbruntime‚2795  _runtime‚2795·_ átµî &loss/total_loss‚2.4754628188820296 % _timestamp‚1.7866614470980275e+09 _runtime‚2795.627907937 0loss/FRt2/assignment_loss‚0.6059780112998514 /loss/FRt2/detection_loss‚0.5009693903871429 epoch‚26 trainer/global_step‚10449 _step‚861 0loss/FRt1/assignment_loss‚0.7742517669473604 /loss/FRt1/detection_loss‚0.5942636502476749&5Á3y¶ó trainer/global_step‚10449 _step‚861 0loss/FRt1/assignment_loss‚0.7742517669473604 /loss/FRt1/detection_loss‚0.5942636502476749 &loss/total_loss‚2.4754628188820296 % _timestamp‚1.7866614470980275e+09 _runtime‚2795.627907937 0loss/FRt2/assignment_loss‚0.6059780112998514 /loss/FRt2/detection_loss‚0.5009693903871429 epoch‚26ÝÈ ¸®•·‚(:Š ДùÓóÇ‹á gpu.0.gpu‚59.0 gpu.0.memory‚11, gpu.0.memoryAllocated‚7.0313262939453125) gpu.0.memoryAllocatedBytes‚ 6039863296 gpu.0.temp‚35.0 gpu.0.powerWatts‚205.275' gpu.0.enforcedPowerLimitWatts‚500.0 gpu.0.powerPercent‚41.055 gpu.0.smClock‚1410 gpu.0.memoryClock‚1593$ gpu.0.correctedMemoryErrors‚1955# gpu.0.uncorrectedMemoryErrors‚0sÎb —¸‚(:Œ ДùÓ·¿Àã( disk.nvme0c0n1.out‚56431.59912109375 disk.sda1.out‚0 disk.drbd2165.out‚0) proc.memory.availableMB‚ 5259.53515625 cpu‚27.17502003565839 disk.sda2.out‚0 network.sent‚ 1.3670792e+07 network.recv‚ 5.276048e+06 disk.sda3.in‚25988.94921875 disk.sda.in‚25988.94921875 disk.sda.out‚ 52.328125 disk.sda2.in‚0 proc.cpu.threads‚17$ disk./.usageGB‚487.6028060913086# disk.nvme0c0n1.in‚ 43561.9921875& disk.nvme0n1.out‚56431.59912109375 disk.dm-0.in‚ 43561.9921875# disk.dm-0.out‚56431.59912109375) proc.memory.percent‚14.46521282196045 disk.sda3.out‚ 52.328125 disk.sda1.in‚0! disk.nvme0n1.in‚ 43561.9921875 disk.dm-1.in‚25988.94921875 disk.dm-1.out‚ 52.328125 disk.drbd2165.in‚0$ memory_percent‚67.89834499359131# proc.memory.rssMB‚ 2369.98046875( disk./.usagePercent‚32.7234334427926:<“2¹- _wandbruntime‚2808  _runtime‚2808Þ©ãZȺ & lr-AdamW/pg1‚0.0007140697706178209 & lr-AdamW/pg2‚0.0007140697706178209 trainer/global_step‚10499 % _timestamp‚1.7866614595046766e+09 _runtime‚2808.034709456 _step‚862Ô× Í»Ç & lr-AdamW/pg2‚0.0007140697706178209 trainer/global_step‚10499 % _timestamp‚1.7866614595046766e+09 _runtime‚2808.034709456 _step‚862 & lr-AdamW/pg1‚0.0007140697706178209Þ¦¥8Ä2¼- _wandbruntime‚2808  _runtime‚2808œ8—U½¸O % _timestamp‚1.7866614595056052e+09 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 2539127269376 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚35250577 PIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚84 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚252477344807424 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚262 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚289098622035968 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 122905600 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚35246290 PCDeviceStatsMonitor.on_train_batch_start/reques³mêÆi$ted_bytes.all.current‚58387380 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚289098681085952 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚286559494766592 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚8814925 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 845178880 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚41348020 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚322 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚262 M@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚59049984 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚38884475 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 831258624 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚41945088 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 2685041504768 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚4287 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚223 UHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚73070592 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 4654602240 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚74135054 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚35250577 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚284864309264384 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17104896 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 5494538240 _step‚863 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚289098622035968 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚38884477 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 5494538240 F?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚60 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 _runtime‚2808.038876357 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚38884477 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚252477417878016 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚262 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 E8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚74130765 JBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚665 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚24209301 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚249792355729408 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 2684989078016 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚322 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 @7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚5264 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚286559511871488 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚41945088 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 4517085184 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 KDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚60 PCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚59049984 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚33024312 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚33024226 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 5368709120 IDeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4289 IDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚24209303 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚20643840 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚4289 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚249792376373248 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚52426752 G>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4876 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚209 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 D;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚5264 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17104896 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 2539127269376 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚429 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 5368709120 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 125829120 KBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚4876 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚8815009 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 IBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚60 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 122306526 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 125829120 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚4287 IBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚86 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚284864326303744 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 125829120 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 2533841488951 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚289098681085952 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚287398209140715 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 5368709120 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚61478912 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 5494538240 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 4552982528 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚286559511871488 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 2539169214464 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0#JpÃ'¾½O WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17104896 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 5494538240 _step‚863 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚289098622035968 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚38884477 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 5494538240 F?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚60 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 _runtime‚2808.038876357 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚38884477 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚252477417878016 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚262 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 E8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚74130765 JBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚665 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚24209301 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚249792355729408 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 2684989078016 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚322 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 @7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚5264 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚286559511871488 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚41945088 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 4517085184 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 KDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚60 PCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚59049984 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚33024312 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚33024226 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 5368709120 IDeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4289 IDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚24209303 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚20643840 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚4289 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚249792376373248 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚52426752 G>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4876 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚209 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 D;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚5264 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17104896 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 2539127269376 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚429 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 5368709120 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 125829120 KBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚4876 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚8815009 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 IBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚60 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 122306526 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 125829120 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚4287 IBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚86 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚284864326303744 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 125829120 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 2533841488951 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚289098681085952 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚287398209140715 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 5368709120 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚61478912 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 5494538240 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 4552982528 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚286559511871488 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 2539169214464 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 % _timestamp‚1.7866614595056052e+09 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 2539127269376 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚35250577 PIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚84 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚252477344807424 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚262 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚289098622035968 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 122905600 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚35246290 PCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚58387380 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚289098681085952 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚286559494766592 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚8814925 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 845178880 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚41348020 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚322 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚262 M@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚59049984 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚38884475 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 831258624 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚41945088 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 2685041504768 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚4287 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚223 UHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚73070592 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 4654602240 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚74135054 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚35250577 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚284864309264384 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0ßÒÒ¡Ž2¿- _wandbruntime‚2808  _runtime‚2808-\*ÅÈ&ÀÂM XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 2539358338048 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚8815733 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚35253870 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 5494538240 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 G@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚60 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚35249577 >5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5264 SFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚73067520 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚284890512097280 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 D=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚60 % _timestamp‚1.7866614596720781e+09 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚38888028 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚35253870 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17104896 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 4552982528 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 G@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚72 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚33027239 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚24211508 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 5494538240 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 2685287745024 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 125829120 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚38888030 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚223 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚262 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚41948160 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 122905600 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 2539400286208 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 122905600 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚249815403102208 NADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚59053056 _runtime‚2808.202416703 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚4293 B9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚5264 H@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚665 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521206 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 5368709120 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚4295 EDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚74141900 K>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59053056 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654602240 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚74137605 I@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚4876 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚429 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚287424567119483 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 2534113409631 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 5494538240 DDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 125829120 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚322 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚74137605 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚322 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚74141900 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚35249577 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚41948160 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 831258624 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 4517085184 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 _step‚864 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 5368709120 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚24211506 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 845178880 NADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚58387428 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 2534072061563 trainer/global_step‚10499 P6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚322 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚262 IBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚60 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚289125234495488 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 2539358338048 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚289125175442432 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚286585817104384 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 125829120 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚262 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 4552982528 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚20643840 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚322 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 4654602240 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚4293 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚52423680 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚33027311 NGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚70 B5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5264 SFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚73067520 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚284890512097280 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 D=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚60 % _timestamp‚1.7866614596720781e+09 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚38888028 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚35253870 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17104896 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 4552982528 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 G@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚72 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚33027239 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚24211508 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 5494538240 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 2685287745024 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 125829120 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚38888030 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚223 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚262 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚41948160 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 122905600 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 2539400286208 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 122905600 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚249815403102208 NADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚59053056 _runtime‚2808.202416703 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚4293 B9DeviceStad+`tsMonitor.on_train_batch_end/allocation.all.peak‚5264 H@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚665 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521206 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 5368709120 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚4295 EDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚74141900 K>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59053056 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654602240 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚74137605 I@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚4876 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚429 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚287424567119483 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 2534113409631 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 5494538240 DDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 125829120 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚322 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚74137605 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚322 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚74141900 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚35249577 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚41948160 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 831258624 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 4517085184 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 _step‚864 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 5368709120 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚24211506 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 845178880 NADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚58387428 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 2534072061563 trainer/global_step‚10499 P6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚322 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚262 IBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚60 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚289125234495488 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 2539358338048 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚289125175442432 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚286585817104384 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 125829120 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚262 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 4552982528 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚20643840 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚322 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 4654602240 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚4293 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚52423680 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚33027311 NGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚70 BDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4876 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17104896 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚39062125 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 125829120 E8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚74473517 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚61478912 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚288719060628123 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚39062127 M@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚59049984 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 5368709120 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚223 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚287875628761088 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 2550714035200 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚35411392 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 2545403299559 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚287875611656192 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚286173598941184 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚4287 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 4654602240 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚250944724369408 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚8855728 BDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚35411392 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚262 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚290426384741376 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 UHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚66779136 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚288719002240743 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 5368709120 F?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚60 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 4552982528 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 2550755980288 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 122905600 JBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚665 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 125829120 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 845178880 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 2545444647579 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 5494538240 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17104896 PIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚76 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 4628521206 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚35415679 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚322 IDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚290426325691392 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚41945088 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚24319553 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚209 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚253642009752576 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚35415679 D;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚5264 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 PCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚58387380 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 KDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚60 IBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚60 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4289 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚39062127 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 831258624 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 4517085184 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 5368709120 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 2550755980288 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 2697285383168 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚322 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 5494538240mÛÂÂ'̼O ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚287875628761088 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚41945088 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚46135296 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚322 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚24319551 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 122905600 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚74477806 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 2697331518464 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚4289 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚429 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 125829120 _runtime‚ 2820.92298336 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚290426325691392 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 4552982528 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚4287 IBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚78 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 4654602240 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚322 _step‚867 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚41348020 G>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4876 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17104896 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚39062125 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 125829120 E8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚74473517 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚61478912 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚288719060628123 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚39062127 M@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚59049984 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 5368709120 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚223 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚287875628761088 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 2550714035200 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚35411392 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 2545403299559 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚287875611656192 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚286173598941184 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚4287 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 4654602240 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚250944724369408 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚8855728 BDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚35411392 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚262 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚290426384741376 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 UHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚66779136 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚288719002240743 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 5368709120 F?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚60 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 4552982528 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 2550755980288 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 122905600 JBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚665 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 125829120 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 845178880 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 2545444647579 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 5494538240 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17104896 PIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚76 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 4628521206 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚35415679 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚322 IDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚290426325691392 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚41945088 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚24319553 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚209 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚253642009752576 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚35415679 D;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚5264 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 PCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚58387380 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 KDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚60 IBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚60 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4289 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚39062127 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 831258624 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 4517085184 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 5368709120 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 2550755980288 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 2697285383168 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚322 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 5494538240ãë¥/a2Í- _wandbruntime‚2821  _runtime‚2821Qüña• ÎÂM GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521206 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚262 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚24321758 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17104896 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚74480357 I@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚4876 NGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚66 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 845178880 EDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 5368709120 D=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚60 _step‚868 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚35414679 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 2550945103872 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚290452879097856 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚35414679 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚8856514 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚209 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 2697527596032 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 5494538240 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚35418972 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚223 BDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚33178270 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654602240 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚4293 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 4552982528 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚20643840 NADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚58387428 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 125829120 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚39065678 PDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59053056 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚61478912 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 2550987052032 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 2697580019712 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚52423680 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 2545675220239 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 5368709120 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚39065680 G@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚68 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 5368709120 >5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5264 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚287901933993984 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚41948160 B9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚5264 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚35418972 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 4654602240 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚287901933993984 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 5494538240 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚429 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚290452938150912 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚8856580 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚288745476994319 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 122905600 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚286199784734720 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚74480357 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 4552982528 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚290452938150912 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚286199801774080 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚322 _runtime‚2821.089366755 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚253665372405760 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚4295 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 122905600 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚33178338 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 NADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚59053056 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚287901951098880 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚250967771742208 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚322 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17104896 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 125829120 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚24321756 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚74484652 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚41948160 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚39065678 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 5494538240 G@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚60 E8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚288745418606891 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 2550945103872 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 831258624 IBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚60 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚39065680 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚4293 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 % _timestamp‚1.7866614725589712e+09 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚74484652 H@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚665 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 2545633872171 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 122306526 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 125829120 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 DDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 DDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521206 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚262 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚24321758 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17104896 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚74480357 I@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚4876 NGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚66 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 845178880 EDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 5368709120 D=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚60 _step‚868 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚35414679 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 2550945103872 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚290452879097856 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚35414679 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚8856514 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚209 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 2697527596032 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 5494538240 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚35418972 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚223 BDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚33178270 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654602240 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚4293 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 4552982528 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚20643840 NADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚58387428 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 125829120 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚39065678 PDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59053056 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚61478912 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 2550987052032 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 2697580019712 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚52423680 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 2545675220239 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 5368709120 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚39065680 G@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚68 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 5368709120 >5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5264 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚287901933993984 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚41948160 B9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚5264 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚35418972 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 4654602240 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚287901933993984 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 5494538240 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚429 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚290452938150912 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚8856580 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚288745476994319 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 122905600 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚286199784734720 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚74480357 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 4552982528 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚290452938150912 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚286199801774080 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚322 _runtime‚2821.089366755 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚253665372405760 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚4295 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 122905600 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚33178338 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 NADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚59053056 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚287901951098880 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚250967771742208 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚322 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17104896 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 125829120 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚24321756 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚74484652 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚41948160 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚39065678 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 5494538240 G@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚60 E8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚288745418606891 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 2550945103872 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 831258624 IBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚60 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚39065680 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚4293 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 % _timestamp‚1.7866614725589712e+09 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚74484652 H@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚665ä܇2Ð- _wandbruntime‚2821  _runtime‚2821'qeuÑï %loss/total_loss‚2.192514321720423 epoch‚26 trainer/global_step‚10549 % _timestamp‚1.7866614725601132e+09 _runtime‚2821.089947081 0loss/FRt1/assignment_loss‚0.6624681628105499 0loss/FRt1/detection_loss‚0.49294200247248504 0loss/FRt2/detection_loss‚0.46064100976785527 _step‚869 0loss/FRt2/assignment_loss‚0.5764631466695326GçœzÒô _runtime‚2821.089947081 0loss/FRt1/assignment_loss‚0.6624681628105499 0loss/FRt1/detection_loss‚0.49294200247248504 0loss/FRt2/detection_loss‚0.46064100976785527 _step‚869 0loss/FRt2/assignment_loss‚0.5764631466695326 %loss/total_loss‚2.192514321720423 epoch‚26 trainer/global_step‚10549 % _timestamp‚1.7866614725601132e+09å_Do2Ó- _wandbruntime‚2833  _runtime‚2833‡øqÈÔ trainer/global_step‚10599 % _timestamp‚1.7866614848495762e+09 _runtime‚2833.379581218 _step‚870 & lr-AdamW/pg1‚0.0007020673026432579 & lr-AdamW/pg2‚0.0007020673026432579ÍZtÍÕÇ % _timestamp‚1.7866614848495762e+09 _runtime‚2833.379581218 _step‚870 & lr-AdamW/pg1‚0.0007020673026432579 & lr-AdamW/pg2‚0.0007020673026432579 trainer/global_step‚10599æƒÆï°2Ö- _wandbruntime‚2833  _runtime‚28334š€¾'׸O YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚291754088396800 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚41945088 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 5368709120 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚289191745650688 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 4654602240 UHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚66779136 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 2709629635584 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 4628521206 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 E8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚74816269 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚8896242 PCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚58387380 IBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚72 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚35580781 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚289191728545792 PCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚59049984 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 2562300801024 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚4287 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚287482888617984 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17104896 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4289 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 122905600 D;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚5264 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚39239775 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚35580781 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 4517085184 F?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚60 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚289191745650688 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 845178880 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 831258624 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚46135296 G>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4876 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚322 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚252097093009408 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚41348020 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚262 IBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚60 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚33325973 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚39239775 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚41945088 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 2562300801024 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 4552982528 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚35576494 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚290039912115531 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 5494538240 IDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 BDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚291754029346816 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 KDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚60 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 4654602240 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚24429801 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚209 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 5368709120 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 125829120 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 2562342746112 JBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚665 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚4287 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚291754029346816 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 5494538240 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚223 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 122905600 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚20643840 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 125829120 _runtime‚2833.381862678 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚262 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 IDeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4289 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 122905600 D;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚5264 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚39239775 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚35580781 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 4517085184 F?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚60 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚289191745650688 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 845178880 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 831258624 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚46135296 G>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4876 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚322 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚252097093009408 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚41348020 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚262 IBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚60 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚33325973 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚39239775 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚41945088 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 2562300801024 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 4552982528 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚35576494 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚290039912115531 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 5494538240 IDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 BDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚291754029346816 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 KDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚60 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 4654602240 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚24429801 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚209 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 5368709120 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 125829120 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 2562342746112 JBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚665 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚4287 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚291754029346816 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 5494538240 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚223 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 122905600 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚20643840 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 125829120 _runtime‚2833.381862678 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚262 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 IDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59053056 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 4552982528 DDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654602240 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚24432006 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚41348068 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 5368709120 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚35584074 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 2709833241600 NADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚58387428 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 5494538240 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 2709873082368 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚290066270094299 IBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚60 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚74827404 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚289218067988480 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚74823109 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚8896975 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚290066328481727 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚287509091450880 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 5494538240 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚322 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 4552982528 G@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚69 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 5494538240 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚322 _step‚872 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 2562531869696 I@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚4876 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚33328981 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚429 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521206 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 2557237030847 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚262 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚291780641806336 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 125829120 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 D=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚60 >5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5264 EDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17104896 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚39243330 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚35579781 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚8897042 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 2557195682779 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚291780582753280 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚24432008 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚39840768 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 5368709120 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 125829120 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚262 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 4654602240 EDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚39243328 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚74827404 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚223 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚209 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 122306526 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚39243330 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 122905600 B6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚322 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 _runtime‚2833.545612777ã ‡cË&ÛÅM \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 2562573817856 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 H@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚665 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚252120161026048 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 125829120 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚289218067988480 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚41948160 NGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚67 SFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚60484608 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚289218050883584 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 4517085184 PDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654602240 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚24432006 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚41348068 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 5368709120 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚35584074 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 2709833241600 NADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚58387428 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 5494538240 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 2709873082368 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚290066270094299 IBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚60 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚74827404 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚289218067988480 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚74823109 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚8896975 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚290066328481727 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚287509091450880 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 5494538240 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚322 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 4552982528 G@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚69 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 5494538240 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚322 _step‚872 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 2562531869696 I@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚4876 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚33328981 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚429 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521206 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 2557237030847 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚262 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚291780641806336 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 125829120 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 D=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚60 >5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5264 EDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17104896 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚39243330 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚35579781 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚8897042 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 2557195682779 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚291780582753280 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚24432008 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚39840768 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 5368709120 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 125829120 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚262 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 4654602240 EDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚39243328 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚74827404 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚223 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚209 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 122306526 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚39243330 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 122905600 B6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚322 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 _runtime‚2833.545612777 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚4295 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚289218050883584 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚252120140382208 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚287509074411520 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚291780641806336 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17104896 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚4293 K>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59053056 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 4552982528 DDeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4289 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚8937121 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚253249461649408 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 5494538240 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚293081733002240 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 845178880 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 _runtime‚2846.092885396 _step‚875 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 122306526 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚293081733002240 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17104896 PIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚59 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚4289 F>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚35741596 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚290507862540288 UHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚70973440 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 831258624 PCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚58387380 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 KBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚4876 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 2721853863424 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 2568568268795 % _timestamp‚1.7866614975596511e+09 IDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 122905600 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚429 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚24540053 BDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4876 M@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚59049984 PCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚59049984 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚35741596 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 4517085184 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚293081792052224 D;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚5264 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 2721904193024 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚61478912 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 2568526920775 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚290507862540288 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚291360763602939 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚35745883 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚4287 @7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚5264 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 4552982528 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 2573887566848 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚209 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 125829120 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚322€Ÿ…’Ã'æ½O GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 2573887566848 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚253249482293248 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 4628521206 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 E8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚75159021 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 4654602240 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚322 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚41945088 IDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 122905600 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚429 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚24540053 BDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4876 M@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚59049984 PCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚59049984 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚35741596 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 4517085184 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚293081792052224 D;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚5264 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 2721904193024 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚61478912 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 2568526920775 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚290507862540288 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚291360763602939 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚35745883 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚4287 @7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚5264 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 4552982528 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 2573887566848 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚209 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 125829120 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚322 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 2573929511936 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 5368709120 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4289 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚8937121 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚253249461649408 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 5494538240 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚293081733002240 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 845178880 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 _runtime‚2846.092885396 _step‚875 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 122306526 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚293081733002240 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17104896 PIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚59 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚4289 F>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚35741596 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚290507862540288 UHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚70973440 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 831258624 PCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚58387380 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 KBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚4876 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 2721853863424 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 2568568268795 % _timestamp‚1.7866614975596511e+09 I8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521206 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚35749176 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚41948160 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚35749176 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 125829120 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚75170156 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 4552982528 H@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚665 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚4293 I@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚4876 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚24542256 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚255994611783168 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 122306526 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚290534167773184 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚24542258 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚41937920 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚39420980 NADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚59053056 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 5368709120 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚262 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 2574118635520 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚61478912 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚322 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 4517085184 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚41948160 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚290534167773184 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚8937923 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 2568757493387 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 5368709120 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 125829120 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚322 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚322 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚293108345461760 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 2574160583680 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17104896 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 P5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5264 EDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59053056 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 122905600 EDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚75170156 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚20643840 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚288818381127680 IBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚60 G@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚60 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚4293 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 122905600 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚429 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚253272509022208 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚291387179969135 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚290534184878080 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚33480179 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚209 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 845178880 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚322 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚39420978 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654602240 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚39420980 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 D6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚322 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 4517085184 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚41948160 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚290534167773184 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚8937923 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 2568757493387 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 5368709120 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 125829120 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚322 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚322 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚293108345461760 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 2574160583680 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17104896 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 P5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5264 EDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59053056 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 122905600 EDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚75170156 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚20643840 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚288818381127680 IBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚60 G@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚60 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚4293 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 122905600 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚429 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚253272509022208 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚291387179969135 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚290534184878080 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚33480179 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚209 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 845178880 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚322 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚39420978 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654602240 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚39420980 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 D8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521206 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚35749176 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚41948160 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚35749176 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 125829120 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚75170156 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 4552982528 H@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚665 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚4293 I@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚4876 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚24542256 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚255994611783168 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 122306526 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚290534167773184 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚24542258 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚41937920 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚39420980 NADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚59053056 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 5368709120 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚262ìÛK N2ê- _wandbruntime‚2846  _runtime‚2846¹mIˆtëî % _timestamp‚1.7866614977300901e+09 _runtime‚2846.259955534 /loss/FRt2/assignment_loss‚0.611890966956791 _step‚877 0loss/FRt1/assignment_loss‚0.5654585807690539 /loss/FRt1/detection_loss‚0.4887482647828668 0loss/FRt2/detection_loss‚0.46771615682482887 &loss/total_loss‚2.1338139693335405 epoch‚26 trainer/global_step‚10649Wñ™íyìó /loss/FRt2/assignment_loss‚0.611890966956791 _step‚877 0loss/FRt1/assignment_loss‚0.5654585807690539 /loss/FRt1/detection_loss‚0.4887482647828668 0loss/FRt2/detection_loss‚0.46771615682482887 &loss/total_loss‚2.1338139693335405 epoch‚26 trainer/global_step‚10649 % _timestamp‚1.7866614977300901e+09 _runtime‚2846.259955534í`÷®•í‚(:Š ý”ùÓâ³íà gpu.0.gpu‚63.0 gpu.0.memory‚12, gpu.0.memoryAllocated‚7.0313262939453125) gpu.0.memoryAllocatedBytes‚ 6039863296 gpu.0.temp‚35.0 gpu.0.powerWatts‚213.875' gpu.0.enforcedPowerLimitWatts‚500.0 gpu.0.powerPercent‚42.775 gpu.0.smClock‚1410 gpu.0.memoryClock‚1593$ gpu.0.correctedMemoryErrors‚1955# gpu.0.uncorrectedMemoryErrors‚0dEí× î‚(:• ý”ùÓÍ™¹ã disk.drbd2165.out‚0( disk.nvme0c0n1.out‚56524.54150390625 disk.sda2.in‚0 disk.sda2.out‚0" disk.nvme0n1.in‚43562.12109375& disk.nvme0n1.out‚56524.54150390625 network.recv‚ 5.351615e+06 disk.dm-0.in‚43562.12109375# disk.dm-0.out‚56524.54150390625 disk.sda.in‚25988.94921875 disk.sda1.in‚0 disk.sda1.out‚0$ memory_percent‚67.89896488189697# proc.memory.rssMB‚ 2369.98046875* disk./.usagePercent‚32.723436514883005$ disk./.usageGB‚487.6028518676758 disk.dm-1.in‚25988.94921875 disk.sda3.in‚25988.94921875 disk.sda3.out‚ 52.9296875$ disk.nvme0c0n1.in‚43562.12109375 disk.sda.out‚ 52.9296875 disk.drbd2165.in‚0) proc.memory.availableMB‚ 5259.43359375) proc.memory.percent‚14.46521282196045 cpu‚27.174025090011213 proc.cpu.threads‚17 network.sent‚ 1.3896317e+07 disk.dm-1.out‚ 52.9296875ø&é2ï- _wandbruntime‚2858  _runtime‚2858¡s_¨ÇðÁ trainer/global_step‚10699 $ _timestamp‚1.786661510019739e+09 _runtime‚2858.551904977 _step‚878 & lr-AdamW/pg1‚0.0006900771489812867 & lr-AdamW/pg2‚0.00069007714898128671§Ê"ÌñÆ $ _timestamp‚1.786661510019739e+09 _runtime‚2858.551904977 _step‚878 & lr-AdamW/pg1‚0.0006900771489812867 & lr-AdamW/pg2‚0.0006900771489812867 trainer/global_step‚10699î.Õ2ò- _wandbruntime‚2858  _runtime‚2858ïО'ó¸O UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚292681556702967 IBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚60 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚291823979429888 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚291823962324992 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚262 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 122306526 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 4654602240 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚322 M@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚59049984 UHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚73070592 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 2580088731383 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 125829120 F>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4289 IBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚79 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 5494538240 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 2585474332672 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 122905600 PIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚77 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 5368709120 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚294409495707648 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 2585516277760 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚61478912 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 5494538240 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚294409495707648 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚294409436657664 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚39595075 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚24650301 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚294409436657664 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚75506062 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚8978223 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚20643840 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 125829120 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚322 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚39595077 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚41945088 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚33628526 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚429 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚209 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚262 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 2585474332672 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 831258624 JBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚665 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚254401850933248 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚35910985 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚4287 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚257135956772352 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚290101485010944 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 5368709120 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚223 @7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚5264 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚35906698 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17104896 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚290101467971584 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚35906698 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 2734178909696 PCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚58387380 KDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚60 % _timestamp‚1.7866615100204144e+09 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚291823962324992 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 122905600 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 2580130079403 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚322 KBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚4876 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 5368709120 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚4289 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 2585516277760 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚4287 IDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4876 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚8978146 PCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚59049984 IDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4876 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚8978146 PCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚59049984 IDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4289 IBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚79 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 5494538240 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 2585474332672 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 122905600 PIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚77 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 5368709120 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚294409495707648 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 2585516277760 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚61478912 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 5494538240 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚294409495707648 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚294409436657664 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚39595075 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚24650301 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚294409436657664 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚75506062 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚8978223 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚20643840 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 125829120 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚322 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚39595077 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚41945088 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚33628526 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚429 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚209 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚262 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 2585474332672 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 831258624 JBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚665 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚254401850933248 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚35910985 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚4287 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚257135956772352 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚290101485010944 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 5368709120 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚223 @7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚5264 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚35906698 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17104896 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚290101467971584 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚35906698 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 2734178909696 PCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚58387380 KDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚60 % _timestamp‚1.7866615100204144e+09 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚291823962324992 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 122905600 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 2580130079403 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚322 KBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚4876 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 5368709120 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚4289 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 2585516277760 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚4287 IDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚294436049117184 H@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚665 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 125829120 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚291850301767680 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚291850301767680 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 2734371186176 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚290127653765120 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 2580319303995 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 >5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5264 EDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59053056 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚294435990064128 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 IBDeviceStatsMonitor.on_train_batch_end/É?°ä segment.small_pool.allocated‚60 NADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚59053056 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 2585747349504 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 845178880 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚262 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚35909985 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚24652508 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚8978956 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 125829120 NGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚67 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚322 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654602240 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 4552982528 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚61478912 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚39598628 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚41948160 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚291850284662784 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 2585705401344 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚33631531 BDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚39598630 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚75512908 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚33631462 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚292707973069115 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 4654602240 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚75508613 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚39598628 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚8979023 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚254424898306048 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚322 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚322 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚291850284662784 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17104896 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521206 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 _runtime‚2858.713196989Š!{èÍ&÷ÇM [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚291850301767680 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚291850301767680 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 2734371186176 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚290127653765120 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 2580319303995 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 >5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5264 EDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59053056 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚294435990064128 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 IBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚60 NADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚59053056 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 2585747349504 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 845178880 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚262 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚35909985 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚24652508 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚8978956 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 125829120 NGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚67 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚322 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654602240 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 4552982528 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚61478912 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚39598628 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚41948160 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚291850284662784 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 2585705401344 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚33631531 BDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚39598630 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚75512908 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚33631462 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚292707973069115 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 4654602240 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚75508613 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚39598628 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚8979023 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚254424898306048 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚322 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚322 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚291850284662784 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17104896 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521206 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 _runtime‚2858.713196989 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚4293 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚24652506 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 122306526 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 5494538240 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 125829120 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 831258624 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚39598630 I@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚4876 G@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚60 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚4295 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 4552982528 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚46132224 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 G@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚69 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚75512908 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 2585705401344 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚41348068 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚35914278 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17104896 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 122905600 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚292708031456543 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚290127670804480 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚294436049117184 H@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚665 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 125829120ð~ók2ø- _wandbruntime‚2858  _runtime‚2858$h]§sùí % _timestamp‚1.7866615101838663e+09 _runtime‚ 2858.71368995 /loss/FRt1/detection_loss‚0.5214914954073389 &loss/total_loss‚2.2123331986900716 epoch‚26 trainer/global_step‚10699 _step‚881 0loss/FRt1/assignment_loss‚0.6471046027925649 0loss/FRt2/assignment_loss‚0.5640386922687778 /loss/FRt2/detection_loss‚0.4796984082213903k[îxúò 0loss/FRt1/assignment_loss‚0.6471046027925649 0loss/FRt2/assignment_loss‚0.5640386922687778 /loss/FRt2/detection_loss‚0.4796984082213903 % _timestamp‚1.7866615101838663e+09 _runtime‚ 2858.71368995 /loss/FRt1/detection_loss‚0.5214914954073389 &loss/total_loss‚2.2123331986900716 epoch‚26 trainer/global_step‚10699 _step‚881ñïNC•û‚(:Š Œ•ùÓò±¡á gpu.0.gpu‚25.0 gpu.0.memory‚6, gpu.0.memoryAllocated‚7.0313262939453125) gpu.0.memoryAllocatedBytes‚ 6039863296 gpu.0.temp‚33.0 gpu.0.powerWatts‚119.593' gpu.0.enforcedPowerLimitWatts‚500.0 gpu.0.powerPercent‚23.9186 gpu.0.smClock‚1410 gpu.0.memoryClock‚1593$ gpu.0.correctedMemoryErrors‚1955# gpu.0.uncorrectedMemoryErrors‚0‹Vc—ü‚(:Œ Œ•ùÓÉàáã proc.cpu.threads‚17 network.sent‚ 1.3964596e+07 disk.sda.out‚ 53.33203125 disk.sda2.out‚0% disk.nvme0n1.out‚56531.2197265625 disk.dm-1.out‚ 53.33203125# proc.memory.rssMB‚ 2369.98046875 disk.sda2.in‚0$ disk./.usageGB‚487.6020698547363 disk.sda3.out‚ 53.33203125$ disk.nvme0c0n1.in‚43579.25390625' disk.nvme0c0n1.out‚56531.2197265625 disk.sda.in‚ 26005.953125 disk.sda1.in‚0" disk.dm-0.out‚56531.2197265625 network.recv‚ 5.376679e+06 disk.sda1.out‚0" disk.nvme0n1.in‚43579.25390625 disk.drbd2165.in‚0( proc.memory.availableMB‚ 5259.3984375) proc.memory.percent‚14.46521282196045) disk./.usagePercent‚32.72338403333849 disk.sda3.in‚ 26005.953125 disk.dm-1.in‚ 26005.953125 disk.drbd2165.out‚0 disk.dm-0.in‚43579.25390625$ memory_percent‚67.89917945861816 cpu‚27.17372258766874M´Êã2ý- _wandbruntime‚2871  _runtime‚2871(w›ãÈþ _runtime‚2871.207729008 _step‚882 & lr-AdamW/pg1‚0.0006840876527787474 & lr-AdamW/pg2‚0.0006840876527787474 trainer/global_step‚10749 % _timestamp‚1.7866615226777146e+09‚g»ÌÍÿÇ & lr-AdamW/pg1‚0.0006840876527787474 & lr-AdamW/pg2‚0.0006840876527787474 trainer/global_step‚10749 % _timestamp‚1.7866615226777146e+09 _runtime‚2871.207729008 _step‚882òZl‰¦2€- _wandbruntime‚2871  _runtime‚2871ò÷Õ¾'¸O G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4289 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚322 KDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚60 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚293140079214592 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 2597061098496 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 4552982528 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 5494538240 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 2597103043584 PCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚59049984 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚293140096319488 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚75848814 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚209 IDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17104896 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 5368709120 C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚4289 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚4287 IBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚73 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚20643840 % _timestamp‚1.7866615226786237e+09 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚295737140313088 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 4552982528 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚295737199363072 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚33779315 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 2746455759872 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚294002408190375 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚41348020 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚255554198929408 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 125829120 _step‚883 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚293140079214592 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 4628521206 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚262 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚33779388 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 122306526 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 5368709120 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚255554219573248 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚262 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 2597061098496 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚36071800 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 831258624 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 5494538240 F?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚60 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 4654602240 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚293140096319488 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚9018764 @7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚5264 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚39772727 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚39772725 KBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚4876 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 2591691890011 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚262 trainer/global_step‚10749 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚4287 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚41945088 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 UHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚66779136 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚294002466577755 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚295737199363072 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚41945088 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚223 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚24760551 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚258300675333120 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚291410757648384 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 5494538240 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 125829120 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 E8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚75844525 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 M@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚59049984 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 845178880 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚322 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚24760553 IDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4876 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 122905600 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 122905600 D;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚5264 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚9018835 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚429 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚291410774687744 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 2591650541991 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚322 _runtime‚2871.211876593 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚295737140313088 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚39772725 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚46135296 IBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚60 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚39772727 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚36076087 BDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0†æÎÊô‚½O F>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17104896 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 5368709120 C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚4289 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚4287 IBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚73 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚20643840 % _timestamp‚1.7866615226786237e+09 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚295737140313088 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 4552982528 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚295737199363072 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚33779315 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 2746455759872 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚294002408190375 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚41348020 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚255554198929408 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 125829120 _step‚883 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚293140079214592 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 4628521206 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚262 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚33779388 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 122306526 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 5368709120 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚255554219573248 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚262 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 2597061098496 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚36071800 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 831258624 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 5494538240 F?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚60 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 4654602240 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚293140096319488 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚9018764 @7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚5264 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚39772727 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚39772725 KBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚4876 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 2591691890011 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚262 trainer/global_step‚10749 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚4287 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚41945088 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 UHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚66779136 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚294002466577755 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚295737199363072 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚41945088 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚223 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚24760551 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚258300675333120 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚291410757648384 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 5494538240 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 125829120 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 E8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚75844525 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 M@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚59049984 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 845178880 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚322 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚24760553 IDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4876 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 122905600 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 122905600 D;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚5264 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚9018835 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚429 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚291410774687744 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freÎXçßÏed‚ 2591650541991 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚322 _runtime‚2871.211876593 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚295737140313088 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚39772725 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚46135296 IBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚60 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚39772727 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚36076087 BDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4289 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚322 KDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚60 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚293140079214592 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 2597061098496 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 4552982528 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 5494538240 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 2597103043584 PCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚59049984 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚293140096319488 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚75848814 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚209 IDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚262 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚75855660 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚39776280 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 122905600 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 2597334115328 EDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚39776278 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚75855660 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚258323903054848 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 845178880 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚322 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚4295 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚36075087 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚295763752772608 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 2746700787712 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 2591881114603 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚322 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17104896 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 5368709120 _runtime‚2871.373359743 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚293166418657280 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚41948160 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚33782401 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 4517085184 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 5368709120 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 2597334115328 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 2591922462671 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚41348068 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚223 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚294028882943951 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚39776280 K>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59053056 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚33782325 NADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚58387428 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521206 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚75851365 B9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚5264 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 2597292167168 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚9019569 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚209 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 % _timestamp‚1.7866615228430386e+09 _step‚884 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚295763693719552 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚293166401552384 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚294028824556523 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 >5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5264 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 4654602240 NADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚59053056 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚39776278 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚258323967733760 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 831258624 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚291436943441920 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 125829120 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚36079380 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚24762758 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚9019643 SFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚64678912 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 4552982528 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚61478912 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚293166418657280 D=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚60 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚44035072 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 125829120 EDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654602240 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 4552982528 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 2597292167168 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 125829120 DDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚75851365 I@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚4876 trainer/global_step‚10749 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17104896 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚41948160 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚262 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚36079380 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚322 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚291436960481280 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 122306526 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚293166401552384 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚4293 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 P8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 125829120 EDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654602240 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 4552982528 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 2597292167168 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 125829120 DDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚75851365 I@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚4876 trainer/global_step‚10749 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17104896 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚41948160 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚262 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚36079380 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚322 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚291436960481280 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 122306526 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚293166401552384 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚4293 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 PDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚262 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚75855660 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚39776280 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 122905600 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 2597334115328 EDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚39776278 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚75855660 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚258323903054848 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 845178880 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚322 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚4295 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚36075087 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚295763752772608 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 2746700787712 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 2591881114603 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚322 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17104896 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 5368709120 _runtime‚2871.373359743 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚293166418657280 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚41948160 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚33782401 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 4517085184 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 5368709120 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 2597334115328 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 2591922462671 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚41348068 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚223 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚294028882943951 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚39776280 K>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59053056 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚33782325 NADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚58387428 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521206 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚75851365 B9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚5264 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 2597292167168 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚9019569 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚209 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 % _timestamp‚1.7866615228430386e+09 _step‚884 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚295763693719552 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚293166401552384 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚294028824556523 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 >5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5264 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 4654602240 NADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚59053056 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚39776278 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚258323967733760 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 831258624 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚291436943441920 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 125829120 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚36079380 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚24762758 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚9019643 SFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚64678912 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 4552982528 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚61478912 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚293166418657280 D=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚60 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚44035072ô²‚»32†- _wandbruntime‚2871  _runtime‚2871sh¼2r‡ì epoch‚26 trainer/global_step‚10749 % _timestamp‚1.7866615228440754e+09 _runtime‚ 2871.37423692 0loss/FRt2/assignment_loss‚0.5545445472084993 /loss/FRt1/detection_loss‚0.5434724199327544 /loss/FRt2/detection_loss‚0.4963789274011667 %loss/total_loss‚2.224250433385274 _step‚885 0loss/FRt1/assignment_loss‚0.6298545388428536Áˆ<\wˆñ _step‚885 0loss/FRt1/assignment_loss‚0.6298545388428536 epoch‚26 trainer/global_step‚10749 % _timestamp‚1.7866615228440754e+09 _runtime‚ 2871.37423692 0loss/FRt2/assignment_loss‚0.5545445472084993 /loss/FRt1/detection_loss‚0.5434724199327544 /loss/FRt2/detection_loss‚0.4963789274011667 %loss/total_loss‚2.224250433385274õÖB€¾–‰‚(:‹ ›•ùÓ–Ë‹á gpu.0.gpu‚64.0 gpu.0.memory‚12, gpu.0.memoryAllocated‚7.0313262939453125) gpu.0.memoryAllocatedBytes‚ 6039863296 gpu.0.temp‚34.0 gpu.0.powerWatts‚222.771' gpu.0.enforcedPowerLimitWatts‚500.0 gpu.0.powerPercent‚44.5542 gpu.0.smClock‚1410 gpu.0.memoryClock‚1593$ gpu.0.correctedMemoryErrors‚1955# gpu.0.uncorrectedMemoryErrors‚0nh%>Š‚(:’ ›•ùÓ„âã disk.sda2.out‚0 network.sent‚ 1.4032165e+07 network.recv‚ 5.401542e+06 disk.sda.out‚ 53.70703125 disk.sda2.in‚0% disk.nvme0n1.out‚56575.8173828125 disk.dm-1.in‚26968.33203125# proc.memory.rssMB‚ 2369.98046875 disk.sda3.in‚26968.33203125 disk.sda1.in‚0" disk.nvme0n1.in‚43583.56640625 disk.dm-0.in‚43583.56640625$ memory_percent‚67.89841651916504 cpu‚27.17351480059297$ disk./.usageGB‚487.1313934326172 disk.sda3.out‚ 53.70703125$ disk.nvme0c0n1.in‚43583.56640625' disk.nvme0c0n1.out‚56575.8173828125 disk.dm-1.out‚ 53.70703125 disk.drbd2165.in‚0" disk.dm-0.out‚56575.8173828125 proc.cpu.threads‚17) disk./.usagePercent‚32.69179654372625 disk.sda.in‚26968.33203125 disk.sda1.out‚0 disk.drbd2165.out‚0( proc.memory.availableMB‚ 5259.5234375) proc.memory.percent‚14.46521282196045qòº¢2‹- _wandbruntime‚2883  _runtime‚2883ò¾óȌ & lr-AdamW/pg2‚0.0006781023900031027 trainer/global_step‚10799 % _timestamp‚1.7866615348957462e+09 _runtime‚2883.425837561 _step‚886 & lr-AdamW/pg1‚0.0006781023900031027þâ• ÍÇ % _timestamp‚1.7866615348957462e+09 _runtime‚2883.425837561 _step‚886 & lr-AdamW/pg1‚0.0006781023900031027 & lr-AdamW/pg2‚0.0006781023900031027 trainer/global_step‚10799öík}2Ž- _wandbruntime‚2883  _runtime‚2883¡î‰Á¸O _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 2758687433728 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 2603253700619 IBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚71 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚24870801 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚259465323879424 E8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚76187277 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚39950375 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚76191566 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚41945088 IDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚297064843968512 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚294456196104192 PCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚59049984 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚20643840 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 5494538240 F?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚60 F>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 2608647864320 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚223 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚9059160 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚9059091 @7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚5264 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 122905600 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 4552982528 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚41945088 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚292720064364544 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚39950377 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 UHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚68876288 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 2758735666176 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 4628521206 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚292720047325184 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 125829120 _step‚887 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚256706588213248 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 122905600 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚61478912 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 125829120 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚262 SDDeviceStaú8¯IýtsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 5494538240 trainer/global_step‚10799 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚36236902 M@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚59049984 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚36241189 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚256706567569408 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚41348020 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17104896 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚33929892 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚322 IBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚60 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17104896 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚36236902 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚209 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚48232448 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 % _timestamp‚1.7866615348967319e+09 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚297064843968512 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚24870803 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 _runtime‚2883.429881108 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚294456213209088 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚429 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚295323318065163 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚295323259677783 PIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚69 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 IDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4876 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚294456213209088 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚4287 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 5494538240 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚262 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚4287 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 5368709120 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 5368709120 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 125829120 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚39950377 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚262 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚36241189 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4289 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚39950375 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚297064903018496{ýs£Ã'½O OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 UHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚68876288 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 2758735666176 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 4628521206 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚292720047325184 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 125829120 _step‚887 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚256706588213248 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 122905600 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚61478912 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 125829120 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚262 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 5494538240 trainer/global_step‚10799 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚36236902 M@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚59049984 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚36241189 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚256706567569408 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚41348020 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17104896 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚33929892 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚322 IBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚60 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17104896 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚36236902 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚209 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚48232448 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 % _timestamp‚1.7866615348967319e+09 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚297064843968512 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚24870803 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 _runtime‚2883.429881108 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚294456213209088 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚429 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚295323318065163 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚295323259677783 PIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚69 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 IDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4876 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚294456213209088 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚4287 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 5494538240 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚262 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚4287 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 5368709120 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 5368709120 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 125829120 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚39950377 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚262 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚36241189 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4289 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚39950375 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚297064903018496 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 2758687433728 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 2603253700619 IBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚71 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚24870801 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚259465323879424 E8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚76187277 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚39950375 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚76191566 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚41945088 IDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚297064843968512 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚294456196104192 PCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚59049984 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚20643840 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 5494538240 F?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚60 F>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 2608647864320 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚223 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚9059160 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚9059091 @7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚5264 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 122905600 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 4552982528 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚41945088 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚292720064364544 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚39950377÷u#^2‘- _wandbruntime‚2883  _runtime‚2883dÎÏAÇ&’ÁM JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 5494538240 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚262 D=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚60 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚36240189 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 2758931129344 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 125829120 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚76198412 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚292746250158080 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 2603442925211 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 5368709120 IBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚60 G@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚60 % _timestamp‚1.7866615350586324e+09 _runtime‚ 2883.58897986 G@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚72 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521206 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚4293 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 845178880 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚39953928 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 4654602240 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚297091456428032 I@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚4876 PDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚76198412 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 4517085184 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚322 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚4295 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 5368709120 DDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚33932874 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 2608878932992 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚259488546071552 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚295349734431359 >5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5264 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 2608920881152 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚33932946 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 trainer/global_step‚10799 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 2758979358720 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 5494538240 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 5494538240 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 125829120 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚76194117 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 122905600 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 122306526 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚297091397374976 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 122905600 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚9059938 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚61478912 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚322 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚322 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚294482518441984 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚294482518441984 EDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 NADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚59053056 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚39953930 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚39953930 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 2608878932992 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚295349676043931 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚322 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚262 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚36244482 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 2608920881152 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚36244482 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚9059868 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 125829120 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚39953928 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚24873006 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚41948160 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654602240 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 _step‚888 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚294482535546880 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17104896 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚41948160 SFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚68873216 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚256729635586048 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 E8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 K>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59053056 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚24873008'Œ¼H$"“ÆM ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚20643840 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚292746233118720 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 2603484273279 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚76198412 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 4517085184 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚322 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚4295 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 5368709120 DDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚33932874 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 2608878932992 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚259488546071552 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚295349734431359 >5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5264 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 2608920881152 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚33932946 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 trainer/global_step‚10799 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 2758979358720 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 5494538240 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 5494538240 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 125829120 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚76194117 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 122905600 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 122306526 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚297091397374976 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 122905600 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚9059938 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚61478912 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚322 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚322 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚294482518441984 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚294482518441984 EDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 NADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚59053056 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚39953930 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚39953930 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 2608878932992 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚295349676043931 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚322 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚262 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚36244482 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 2608920881152 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚36244482 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚9059868 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 125829120 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚39953928 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚24873006 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚41948160 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654602240 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 _step‚888 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚294482535546880 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17104896 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚41948160 SFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚68873216 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚256729635586048 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 E8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 K>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59053056 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚24873008 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 5494538240 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚262 D=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚60 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚36240189 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 2758931129344 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 125829120 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚76198412 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚292746250158080 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 2603442925211 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 5368709120 IBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚60 G@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚60 % _timestamp‚1.7866615350586324e+09 _runtime‚ 2883.58897986 G@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚72 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521206 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚4293 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 845178880 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚39953928 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 4654602240 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚297091456428032 I@DeviceStatsMonitor;?`9¨.on_train_batch_end/allocation.small_pool.peak‚4876 PDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚294410253533184 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 2618840135313 _runtime‚2901.440869627 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚50329600 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚41348020 F?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚60 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚36428483 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚296163267313664 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 122905600 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 4552982528 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚260933253765120 M@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚59049984 F>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 PCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚58387380 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 2618881483333 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚262 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚209 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 4517085184 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 5494538240 % _timestamp‚1.7866615529073138e+09 PIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚71 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 5368709120 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚40188633 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚296163250208768 PCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚59049984 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚322 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17104896 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 2624344301568 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚40188631 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 5494538240 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 5368709120 @7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚5264 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚298787552565248 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚9109751 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 2775047247872 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚297029135016517 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 4628521206 C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚4289 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚298787611615232 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚9109680 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚429 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚258158227161088 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 122306526 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚41945088 JBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚665 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚20643840 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚322 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 5368709120 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚322 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚298787611615232 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 125829120 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚36428483 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚4287 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚40188633 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚34131418 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚260933324738560 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚297029076629137 trainer/global_step‚10849 G>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4876 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚40188631 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 2624302356480 KBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚4876 UHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚70973440 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚296163267313664 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4289 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚322 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚262 KDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚60 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚25021665 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 125829120 IBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚60 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚34131345 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚25021667 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚61478912 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚41945088 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚223 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17104896 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚4287 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚36424196 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 122905600 IBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚73 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 845178880 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 831258624 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 2775097577472 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚298787552565248 D;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚5264 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 125829120 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 _step‚892 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 4552982528 IDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚298787552565248 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚9109751 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 2775047247872 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚297029135016517 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 4628521206 C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚4289 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚298787611615232 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚9109680 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚429 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚258158227161088 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 122306526 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚41945088 JBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚665 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚20643840 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚322 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 5368709120 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚322 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚298787611615232 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 125829120 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚36428483 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚4287 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚40188633 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚34131418 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚260933324738560 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚297029076629137 trainer/global_step‚10849 G>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4876 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚40188631 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 2624302356480 KBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚4876 UHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚70973440 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚296163267313664 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4289 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚322 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚262 KDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚60 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚25021665 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 125829120 IBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚60 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚34131345 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚25021667 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚61478912 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚41945088 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚223 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17104896 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚4287 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚36424196 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 122905600 IBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚73 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 845178880 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 831258624 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 2775097577472 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚298787552565248 D;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚5264 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 125829120 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 _step‚892 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 4552982528 IDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚294410253533184 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 2618840135313 _runtime‚2901.440869627 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚50329600 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚41348020 F?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚60 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚36428483 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚296163267313664 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 122905600 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 4552982528 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚260933253765120 M@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚59049984 F>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 PCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚58387380 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 2618881483333 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚262 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚209 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 4517085184 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 5494538240 % _timestamp‚1.7866615529073138e+09 PIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚71 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 5368709120 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚40188633 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚296163250208768 PCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚59049984 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚322 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17104896 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 2624344301568 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚40188631 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 5494538240 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 5368709120üî” ¹2¢- _wandbruntime‚2901  _runtime‚2901ÿ-‰­£ÂM QEDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚9110471 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚258181274533888 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚46132224 I@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚4876 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚298814165024768 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚4295 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚296189572546560 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚262 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 D=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚60 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚296189589651456 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚20643840 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 125829120 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚40192184 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚262 H@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚665 NGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚66 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 845178880 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 5494538240 trainer/global_step‚10849 NADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚59053056 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 2624533425152 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 2619070707925 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚322 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 5494538240 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17104896 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 2624533425152 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚260956614878208 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚258181253890048 NADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚58387428 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 2619112055993 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚40192186 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 122905600 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 _runtime‚2901.599904396 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚4293 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 5368709120 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚36427483 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚298814105971712 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚322 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 4552982528 B9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚5264 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚209 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚296189589651456 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚223 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚41348068 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17104896 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 125829120 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 4654602240 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 831258624 BDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚76623962 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚76619667 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚76623962 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 DDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654602240 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚41948160 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚260956548102144 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 2775294212096 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚61478912 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 K>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59053056 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚41948160 E5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5264 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚262 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚40192184 G@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚68 G@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚60 % _timestamp‚1.7866615530696428e+09 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 4552982528 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 2775340344320 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 2624575373312 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚25023870 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚322 EDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521206 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 122905600 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 5368709120 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚322 PDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚34134341 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 125829120³-E•Í&¤ÇM A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚4295 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚296189572546560 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚262 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 D=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚60 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚296189589651456 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚20643840 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 125829120 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚40192184 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚262 H@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚665 NGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚66 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 845178880 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 5494538240 trainer/global_step‚10849 NADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚59053056 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 2624533425152 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 2619070707925 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚322 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 5494538240 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17104896 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 2624533425152 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚260956614878208 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚258181253890048 NADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚58387428 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 2619112055993 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚40192186 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 122905600 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 _runtime‚2901.599904396 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚4293 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 5368709120 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚36427483 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚298814105971712 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚322 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 4552982528 B9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚5264 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚209 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚296189589651456 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚223 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚41348068 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17104896 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 125829120 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 4654602240 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 831258624 BDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚76623962 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚76619667 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚76623962 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 DDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654602240 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚41948160 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚260956548102144 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 2775294212096 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚61478912 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 K>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59053056 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚41948160 E5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5264 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚262 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚40192184 G@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚68 G@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚60 % _timestamp‚1.7866615530696428e+09 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 4552982528 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 2775340344320 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 2624575373312 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚25023870 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚322 EDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521206 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 122905600 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 5368709120 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚322 PDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚34134341 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 125829120 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚9110471 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚258181274533888 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚46132224 I@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚4876 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚298814165024768ýU`£2¥- _wandbruntime‚2901  _runtime‚2901dšût¦î 0loss/FRt1/assignment_loss‚0.6114602296494083 0loss/FRt1/detection_loss‚0.49357908430035996 &loss/total_loss‚2.0748300085581324 epoch‚27 % _timestamp‚1.7866615530706167e+09 _step‚894 0loss/FRt2/assignment_loss‚0.5164205862808599 /loss/FRt2/detection_loss‚0.4533701083275042 trainer/global_step‚10849 _runtime‚ 2901.60039924yWÔ2y§ó 0loss/FRt1/assignment_loss‚0.6114602296494083 0loss/FRt1/detection_loss‚0.49357908430035996 &loss/total_loss‚2.0748300085581324 epoch‚27 % _timestamp‚1.7866615530706167e+09 _step‚894 0loss/FRt2/assignment_loss‚0.5164205862808599 /loss/FRt2/detection_loss‚0.4533701083275042 trainer/global_step‚10849 _runtime‚ 2901.60039924þ &Nj’¨‚(:‡ ¹•ùÓð‹á gpu.0.gpu‚0.0 gpu.0.memory‚0, gpu.0.memoryAllocated‚7.0313262939453125) gpu.0.memoryAllocatedBytes‚ 6039863296 gpu.0.temp‚35.0 gpu.0.powerWatts‚126.61' gpu.0.enforcedPowerLimitWatts‚500.0 gpu.0.powerPercent‚25.322 gpu.0.smClock‚1410 gpu.0.memoryClock‚1593$ gpu.0.correctedMemoryErrors‚1955# gpu.0.uncorrectedMemoryErrors‚0_;7ž©‚(:“ ¹•ùÓÌÄìâ network.recv‚ 5.456473e+06 disk.sda.in‚26968.33203125 disk.dm-1.in‚26968.33203125 cpu‚27.180248826240017 proc.cpu.threads‚17* disk./.usagePercent‚32.691828800675566% disk./.usageGB‚487.13187408447266 disk.sda3.out‚ 54.32421875 disk.sda.out‚ 54.32421875 disk.sda1.in‚0 disk.sda1.out‚0 disk.sda2.in‚0 disk.sda2.out‚0 network.sent‚ 1.4173834e+07( disk.nvme0c0n1.out‚56782.37158203125" disk.nvme0n1.in‚43583.56640625 disk.dm-1.out‚ 54.32421875# disk.dm-0.out‚56782.37158203125$ memory_percent‚67.60852336883545) proc.memory.availableMB‚ 5307.01953125 disk.sda3.in‚26968.33203125$ disk.nvme0c0n1.in‚43583.56640625& disk.nvme0n1.out‚56782.37158203125 disk.drbd2165.in‚0 disk.drbd2165.out‚0 disk.dm-0.in‚43583.56640625 proc.memory.rssMB‚2322.25) proc.memory.percent‚14.17388916015625\ìº2ª- _wandbruntime‚2913  _runtime‚2913O} È«Â % _timestamp‚1.7866615650274909e+09 _runtime‚2913.557473926 _step‚895 & lr-AdamW/pg1‚0.0006661461021248768 & lr-AdamW/pg2‚0.0006661461021248768 trainer/global_step‚10899)#ô”Í¬Ç & lr-AdamW/pg1‚0.0006661461021248768 & lr-AdamW/pg2‚0.0006661461021248768 trainer/global_step‚10899 % _timestamp‚1.7866615650274909e+09 _runtime‚2913.557473926 _step‚895ÿt† 2­- _wandbruntime‚2913  _runtime‚2913Ϻ³c¾'®¸O L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 4654602240 _runtime‚2913.558620624 D;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚5264 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 122306526 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 5368709120 _step‚896 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 125829120 KDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚60 trainer/global_step‚10899 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚40366281 M@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚59049984 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚40366283 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚40366281 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 2630443293941 JBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚665 KBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚4876 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚223 @7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚5264 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚300115315270656 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 2635931067392 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 5494538240 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚34282457 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚259310595801088 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 831258624 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 2635931067392 IBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚70 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚20643840 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚259310575157248 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚298349928116545 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚262 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚40366283 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚298349986503925 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 4517085184 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚429 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 2635889122304 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚4287 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17104896 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚262097955729920 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 2630401945921 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚322 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 2635889122304 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 2787359928832 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚61478912 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 4628521206 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚297479367098368 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚36593585 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 G>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4876 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚297479384203264 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚36589298 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚322 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚41348020 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚262 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 122905600 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚76959868 IDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17104896 IBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚60 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 4552982528 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 4552982528 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚322 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚297479384203264 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚41945088 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 4654602240 UHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚73070592 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚262097882659328 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 2787307502080 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚36593585 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚300115256220672 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4289 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 125829120 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚41945088 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚52426752 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚300115256220672 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0Å‹ ¯½O C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚4289 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 122905600 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 5494538240 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 5368709120 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚297479367098368 PCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚58387380 E8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚76955579 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚209 BDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17104896 IBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚60 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 4552982528 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 4552982528 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚322 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚297479384203264 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚41945088 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 4654602240 UHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚73070592 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚262097882659328 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 2787307502080 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚36593585 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚300115256220672 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4289 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 125829120 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚41945088 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚52426752 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚300115256220672 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 4654602240 _runtime‚2913.558620624 D;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚5264 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 122306526 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 5368709120 _step‚896 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 125829120 KDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚60 trainer/global_step‚10899 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚40366281 M@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚59049984 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚40366283 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚40366281 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 2630443293941 JBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚665 KBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚4876 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚223 @7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚5264 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚300115315270656 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 2635931067392 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 5494538240 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚34282457 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pood}µ×¶l.current‚2 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚259310595801088 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 831258624 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 2635931067392 IBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚70 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚20643840 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚259310575157248 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚298349928116545 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚262 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚40366283 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚298349986503925 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 4517085184 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚429 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 2635889122304 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚4287 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17104896 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚262097955729920 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 2630401945921 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚322 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 2635889122304 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 2787359928832 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚61478912 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 4628521206 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚297479367098368 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚36593585 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 G>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4876 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚297479384203264 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚36589298 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚322 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚41348020 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚262 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 122905600 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚76959868 I6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚322 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚297505706541056 NGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚67 NADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚58387428 EDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚76966714 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚36592585 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚262121174702592 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 831258624 trainer/global_step‚10899 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚76966714 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚297505706541056 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚259333643173888 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 2630632518533 NADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚59053056 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 _step‚897 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚4295 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚36596878 B9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚5264 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚4293 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚54520832 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 4552982528 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 5368709120 IBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚60 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 122905600 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚295745711964160 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 5368709120 K>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59053056 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚9151303 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚40369834 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚429 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17104896 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 122905600 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚20643840 % _timestamp‚1.7866615651900055e+09 _runtime‚2913.720319983 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚76962419 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 2636120190976 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚25134122 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 122306526 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 2636120190976 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚209 SFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚75164672 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 2787606693376 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚76962419 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚34285492 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚223 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚61478912 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚262 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 2636162139136 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚9151370 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚297505689436160 EDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521206 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 5494538240 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚262 G@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚60 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚40369836 H@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚665 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 845178880 DDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654602240 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚34285423 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚300141809627136 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚25134120 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚262121249867264 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 125829120 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚262 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚36592585 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚40369836 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 125829120 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚322 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 5494538240 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 5368709120 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚300141868680192 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚297505689436160 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 4517085184 >5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5264 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17104896 B8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚300141868680192 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚36596878 G@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚69 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 2787552172544 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚322 D=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚60 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 4552982528 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚2983764028701214„ü¦Í&²ÇM SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 122905600 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚295745711964160 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 5368709120 K>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59053056 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚9151303 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚40369834 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚429 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17104896 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 122905600 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚20643840 % _timestamp‚1.7866615651900055e+09 _runtime‚2913.720319983 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚76962419 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 2636120190976 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚25134122 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 122306526 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 2636120190976 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚209 SFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚75164672 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 2787606693376 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚76962419 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚34285492 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚223 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚61478912 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚262 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 2636162139136 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚9151370 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚297505689436160 EDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521206 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 5494538240 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚262 G@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚60 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚40369836 H@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚665 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 845178880 DDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654602240 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚34285423 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚300141809627136 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚25134120 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚262121249867264 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 125829120 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚262 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚36592585 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚40369836 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 125829120 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚322 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 5494538240 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 5368709120 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚300141868680192 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚297505689436160 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 4517085184 >5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5264 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17104896 B8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚300141868680192 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚36596878 G@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚69 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 2787552172544 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚322 D=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚60 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 4552982528 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚298376402870121 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚295745729003520 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚40369834 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚41948160 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚41348068 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 2636162139136 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 2630673866601 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚322 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚297505706541056 NGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚67 NADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚58387428 EDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚76966714 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚36592585 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚262121174702592 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 831258624 trainer/global_step‚10899 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚76966714 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚297505706541056 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚259333643173888 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 2630632518533 NADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚59053056 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 _step‚897 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚4295 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚36596878 B9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚5264 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚4293 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚54520832 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 4552982528 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 5368709120 IBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚60ÿÂì 2³- _wandbruntime‚2913  _runtime‚2913¿xjt´î &loss/total_loss‚2.0707799611667195 epoch‚27 trainer/global_step‚10899 _runtime‚2913.720827798 _step‚898 0loss/FRt1/assignment_loss‚0.5774117068629242 0loss/FRt2/assignment_loss‚0.5318713319879619 0loss/FRt1/detection_loss‚0.48942307128360835 /loss/FRt2/detection_loss‚0.4720738510322251 $ _timestamp‚1.786661565190979e+09{ëyµó $ _timestamp‚1.786661565190979e+09 &loss/total_loss‚2.0707799611667195 epoch‚27 trainer/global_step‚10899 _runtime‚2913.720827798 _step‚898 0loss/FRt1/assignment_loss‚0.5774117068629242 0loss/FRt2/assignment_loss‚0.5318713319879619 0loss/FRt1/detection_loss‚0.48942307128360835 /loss/FRt2/detection_loss‚0.4720738510322251‚Uí–¶‚(:‹ È•ùÓ¸®á gpu.0.gpu‚64.0 gpu.0.memory‚12, gpu.0.memoryAllocated‚7.0313262939453125) gpu.0.memoryAllocatedBytes‚ 6039863296 gpu.0.temp‚34.0 gpu.0.powerWatts‚239.437' gpu.0.enforcedPowerLimitWatts‚500.0 gpu.0.powerPercent‚47.8874 gpu.0.smClock‚1410 gpu.0.memoryClock‚1593$ gpu.0.correctedMemoryErrors‚1955# gpu.0.uncorrectedMemoryErrors‚0ßoÜ’·‚(:‡ È•ùÓÕ‚Íã disk.sda2.out‚0$ disk.nvme0n1.out‚57004.490234375 disk.drbd2165.in‚0 proc.cpu.threads‚17 network.sent‚ 1.4248884e+07 disk.sda.in‚26968.33203125 disk.dm-1.in‚26968.33203125 disk.dm-1.out‚ 54.44921875! disk.dm-0.out‚57004.490234375 disk.sda3.out‚ 54.44921875$ memory_percent‚67.60857105255127) proc.memory.availableMB‚ 5307.01171875* disk./.usagePercent‚32.725382428150766 disk.sda1.out‚0 disk.sda2.in‚0! disk.nvme0n1.in‚ 43584.3828125 disk.drbd2165.out‚0 disk.dm-0.in‚ 43584.3828125 disk.sda3.in‚26968.33203125& disk.nvme0c0n1.out‚57004.490234375 network.recv‚ 5.489035e+06 disk.sda.out‚ 54.44921875 disk.sda1.in‚0# disk.nvme0c0n1.in‚ 43584.3828125 proc.memory.rssMB‚2322.25) proc.memory.percent‚14.17388916015625 cpu‚27.1801657986225$ disk./.usageGB‚487.6318473815918U1áÚ2¸- _wandbruntime‚2925  _runtime‚2925äÄ~Êȹ trainer/global_step‚10949 % _timestamp‚1.7866615772259865e+09 _runtime‚2925.756244157 _step‚899 & lr-AdamW/pg1‚0.0006601758449523233 & lr-AdamW/pg2‚0.0006601758449523233559꼂 _runtime‚2925.756244157 _step‚899 & lr-AdamW/pg1‚0.0006601758449523233 & lr-AdamW/pg2‚0.0006601758449523233 trainer/global_step‚10949 % _timestamp‚1.7866615772259865e+09ƒ!Fx2»- _wandbruntime‚2925  _runtime‚2925qDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚301442959876096 trainer/global_step‚10949 G>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4876 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚262 IBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚60 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚41945088 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚36758687 KBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚4876 F>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚301443018926080 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17104896 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚9191457 UHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚68876288 BDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 F?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚60 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 5368709120 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚48232448 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚322 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚322 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 4552982528 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚4287 PIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚61 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚260462943797248 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚40543933 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚298795483987968 PCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚58387380 E8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚77298331 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚298795483987968 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚429 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚299670779603953 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 5494538240 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 5368709120 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚34433624 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚25242167 IBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚63 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 831258624 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 2647475888128 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 845178880 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 % _timestamp‚1.7866615772272878e+09 M@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚59049984 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17104896 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚40543931 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 125829120 _step‚900 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ _ôÝ122905600 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚40543933 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚61478912 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚41348020 IDeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4289 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚4287 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚297028832886784 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 125829120 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚298795501092864 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚263262592349184 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 4552982528 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 4517085184 KDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚60 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚40543931 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚301443018926080 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 4654602240 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚9191396 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 2647517833216 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚41945088#™èÓÃ'½½O @7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚5264 PCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚59049984 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚298795501092864 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚77302620 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚20643840 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 4628521206 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚262 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 2647517833216 IDeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4289 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚4287 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚297028832886784 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 125829120 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚298795501092864 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚263262592349184 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 4552982528 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 4517085184 KDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚60 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚40543931 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚301443018926080 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 4654602240 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚9191396 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 2647517833216 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚41945088 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚223 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚299670837991333 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 2642005104549 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚25242165 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 122306526 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 125829120 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚36758687 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚301442959876096 trainer/global_step‚10949 G>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4876 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚262 IBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚60 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚41945088 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚36758687 KBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚4876 F>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚301443018926080 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17104896 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚9191457 UHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚68876288 BDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 F?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚60 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 5368709120 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚48232448 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚322 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚322 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 4552982528 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚4287 PIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚61 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚260462943797248 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚40543933 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚298795483987968 PCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚58387380 E8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚77298331 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚298795483987968 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚429 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚299670779603953 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 5494538240 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 5368709120 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚34433624 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚25242167 IBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚63 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 831258624 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 2647475888128 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 845178880 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 % _timestamp‚1.7866615772272878e+09 M@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚59049984 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17104896 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚40543931 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 125829120 _step‚900 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 122905600 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚40543933 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚61478912 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚41348020 IDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚298821823430656 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚298821823430656 % _timestamp‚1.7866615773906615e+09 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 122905600 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 2647706956800 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 125829120 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚322 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚262 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚40547486 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚301469572335616 I@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚4876 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚41348068 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚41948160 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚260486011813888 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 4517085184 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚322 G@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚60 >5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5264 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚4293 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚298821806325760 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚34436565 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚223 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚9192195 SFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚68873216 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚36761980 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚25244370 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 5494538240 _runtime‚2925.921067878 EDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654602240 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 NADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚58387428 IBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚60 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚36761980 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 122905600 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚77309466 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚299697195970101 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚301469572335616 B9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚5264 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚299697254357529 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚262 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚262 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚77309466 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 2647748904960 EDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 trainer/global_step‚10949 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 2647748904960 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚298821806325760 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 5494538240 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚322 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚77305171 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚48229376 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚322 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚4295 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 5368709120 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚36757687 K>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59053056 NADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚59053056 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 122306526 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 125829120 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚77305171 PDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521206 H@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚665 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 NGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚61 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 2799823297536 D=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚60 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚40547484 DDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚298821823430656 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚298821823430656 % _timestamp‚1.7866615773906615e+09 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 122905600 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 2647706956800 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 125829120 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚322 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚262 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚40547486 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚301469572335616 I@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚4876 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚41348068 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚41948160 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚260486011813888 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 4517085184 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚322 G@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚60 >5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5264 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚4293 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚298821806325760 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚34436565 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚223 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚9192195 SFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚68873216 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚36761980 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚25244370 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 5494538240 _runtime‚2925.921067878 EDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654602240 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 NADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚58387428 IBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚60 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚36761980 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 122905600 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚77309466 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚299697195970101 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚301469572335616 B9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚5264 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚299697254357529 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚262 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚262 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚77309466 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 2647748904960 EDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 trainer/global_step‚10949 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 2647748904960 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚298821806325760 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 5494538240 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚322 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚77305171 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚48229376 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚322 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚4295 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 5368709120 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚36757687 K>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59053056 NADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚59053056 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 122306526 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 125829120 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚77305171 PDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521206 H@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚665 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 NGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚61 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 ]KDeviceStatsMonwÅÕ±Šitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 2799823297536 D=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚60 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚40547484 D!°¾2Ç- _wandbruntime‚2937  _runtime‚2937Œ‘û]¾'ȸO R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚302770663531520 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 2659104599040 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4289 IDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 125829120 E8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚77641083 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚302770722581504 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚264427247620608 _runtime‚2937.873910612 C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚4289 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17104896 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚25352417 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 5368709120 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚322 trainer/global_step‚10999 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚300111617982464 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚41945088 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚40721581 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 125829120 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17104896 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚322 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚300991631091361 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 5494538240 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 4654602240 IBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚60 PIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚58 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚261615333081088 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 2811914539520 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 2659062653952 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚209 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚261615312437248 F?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚60 _step‚904 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 4552982528 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 2659062653952 PCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚59049984 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚77645372 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚39843840 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 845178880 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚20643840 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 5494538240 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 5368709120 G>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4876 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚429 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚25352415 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚298338122563584 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚298338105524224 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚36923789 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚4287 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚264427187132928 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 831258624 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚41945088 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚9232029«ÞO9Ã'ɽO `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚300111617982464 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚34584504 BDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 125829120 E8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚77641083 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚302770722581504 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚264427247620608 _runtime‚2937.873910612 C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚4289 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17104896 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚25352417 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 5368709120 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚322 trainer/global_step‚10999 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚300111617982464 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚41945088 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚40721581 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 125829120 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17104896 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚322 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚300991631091361 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 5494538240 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 4654602240 IBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚60 PIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚58 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚261615333081088 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 2811914539520 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 2659062653952 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚209 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚261615312437248 F?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚60 _step‚904 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 4552982528 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 2659062653952 PCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚59049984 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚77645372 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚39843840 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 845178880 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚20643840 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 5494538240 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 5368709120 G>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4876 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚429 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚25352415 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚298338122563584 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚298338105524224 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚36923789 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚4287 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚264427187132928 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 831258624 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚41945088 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚9232029 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚302770663531520 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 2659104599040 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4289 I8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚322 D=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚60 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚77647923 EDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521206 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 5494538240 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚322 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 K>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59053056 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚209 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 2659335670784 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚36927082 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 831258624 trainer/global_step‚10999 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚301018047457509 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 2653756139749 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 125829120 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 2812115859968 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 5494538240 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚40725134 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚4293 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654602240 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚4293 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 5368709120 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚300137940320256 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚77652218 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚322 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚77652218 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚41948160 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚77647923 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚264450542446080 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚264450475670016 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚261638380453888 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚302797275991040 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚41948160 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 845178880 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚322 IBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚60 _step‚905 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚9232809 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 2653797487817 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 2659293722624 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚40725134 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚46132224 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 125829120 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚9232882 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚262 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 122905600 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 NGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚73 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚36927082 H@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚665 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 125829120 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚262 NADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚59053056 I@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚4876 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚261638359810048 % _timestamp‚1.7866615895011303e+09 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 4552982528 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚25354620 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚20643840 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 BDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 2659293722624 G@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚75 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚34587429 SFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚66776064 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚61478912 D5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5264 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚300137923215360 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚36922789 G@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚60 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚4295 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚300137923215360 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 122905600ERÛ Í&ÌÇM IBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚60 _step‚905 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚9232809 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 2653797487817 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 2659293722624 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚40725134 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚46132224 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 125829120 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚9232882 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚262 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 122905600 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 NGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚73 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚36927082 H@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚665 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 125829120 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚262 NADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚59053056 I@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚4876 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚261638359810048 % _timestamp‚1.7866615895011303e+09 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 4552982528 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚25354620 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚20643840 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 BDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 2659293722624 G@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚75 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚34587429 SFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚66776064 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚61478912 D5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5264 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚300137923215360 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚36922789 G@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚60 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚4295 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚300137923215360 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 122905600 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 4654602240 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚429 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚322 D=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚60 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚77647923 EDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521206 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 5494538240 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚322 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 K>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59053056 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚209 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 2659335670784 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚36927082 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 831258624 trainer/global_step‚10999 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚301018047457509 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 2653756139749 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 125829120 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 2812115859968 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 5494538240 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚40725134 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚4293 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654602240 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚4293 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 5368709120 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚300137940320256 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚77652218 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚322 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚77652218 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚41948160 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚77647923 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚264450542446080 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚264450475670016 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚261638380453888 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚302797275991040 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚41948160 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 845178880 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚322‰÷y¶62Í- _wandbruntime‚2938  _runtime‚2938­B¯ýrÎì /loss/FRt2/detection_loss‚0.5004281615432475 &loss/total_loss‚2.2216360966988042 trainer/global_step‚10999 % _timestamp‚1.7866615895020883e+09 _runtime‚ 2938.03190522 epoch‚27 _step‚906 /loss/FRt1/assignment_loss‚0.660794626460756 0loss/FRt2/assignment_loss‚0.5602835481721352 /loss/FRt1/detection_loss‚0.5001297605226654Ò0ÁkwÏñ /loss/FRt1/detection_loss‚0.5001297605226654 /loss/FRt2/detection_loss‚0.5004281615432475 &loss/total_loss‚2.2216360966988042 trainer/global_step‚10999 % _timestamp‚1.7866615895020883e+09 _runtime‚ 2938.03190522 epoch‚27 _step‚906 /loss/FRt1/assignment_loss‚0.660794626460756 0loss/FRt2/assignment_loss‚0.5602835481721352ŠìB¿Ã–Ђ(:‹ וùÓ˜Íôà gpu.0.gpu‚64.0 gpu.0.memory‚12, gpu.0.memoryAllocated‚7.0313262939453125) gpu.0.memoryAllocatedBytes‚ 6039863296 gpu.0.temp‚35.0 gpu.0.powerWatts‚129.029' gpu.0.enforcedPowerLimitWatts‚500.0 gpu.0.powerPercent‚25.8058 gpu.0.smClock‚1410 gpu.0.memoryClock‚1593$ gpu.0.correctedMemoryErrors‚1955# gpu.0.uncorrectedMemoryErrors‚0§êp™Ñ‚(:Ž וùÓÞ³Ôâ disk.sda2.in‚0 cpu‚27.179863762358295 disk.dm-1.out‚ 54.71484375 disk.sda3.in‚26968.33203125 disk.sda3.out‚ 54.71484375( disk.nvme0c0n1.out‚57121.89990234375! disk.nvme0n1.in‚ 43584.3828125$ memory_percent‚67.60742664337158) proc.memory.percent‚14.17388916015625 network.recv‚ 5.510929e+06) disk./.usagePercent‚32.72537653997748$ disk./.usageGB‚487.6317596435547 disk.drbd2165.out‚0 disk.sda.in‚26968.33203125 disk.sda1.out‚0) proc.memory.availableMB‚ 5307.19921875 proc.memory.rssMB‚2322.25# disk.dm-0.out‚57121.89990234375 disk.sda1.in‚0 disk.sda2.out‚0& disk.nvme0n1.out‚57121.89990234375 proc.cpu.threads‚17# disk.nvme0c0n1.in‚ 43584.3828125 network.sent‚ 1.4314188e+07 disk.dm-1.in‚26968.33203125 disk.drbd2165.in‚0 disk.dm-0.in‚ 43584.3828125 disk.sda.out‚ 54.71484375¯õã&2Ò- _wandbruntime‚2950  _runtime‚2950Øßá¤ÈÓ _step‚907 & lr-AdamW/pg1‚0.0006482530214080175 & lr-AdamW/pg2‚0.0006482530214080175 trainer/global_step‚11049 % _timestamp‚1.7866616017639954e+09 _runtime‚2950.294025621JT*ÍÔÇ _step‚907 & lr-AdamW/pg1‚0.0006482530214080175 & lr-AdamW/pg2‚0.0006482530214080175 trainer/global_step‚11049 % _timestamp‚1.7866616017639954e+09 _runtime‚2950.294025621‹T4‰<2Õ- _wandbruntime‚2950  _runtime‚2950 ÆE_¾'Ö¸O VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 4552982528 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚265591910544384 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 2824164785152 trainer/global_step‚11049 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚37088891 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚37088891 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 5368709120 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 D;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚5264 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 2665128725765 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 125829120 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚301427717767168 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚262767701721088 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚302312540966149 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚37084604 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 2670649419776 IDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚262767681077248 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚304098367186944 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚41348020 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 122905600 PIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚58 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 2824208823296 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 2665087377745 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 125829120 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚262 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17104896 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚37084604 IDeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4289 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚20643840 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 4654602240 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚265591845862400 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚322 _step‚908 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚4287 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚40899233 G>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4876 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚41945088 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚299647395201024 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 5368709120 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 4517085184 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚223 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚304098426236928 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚9272587 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚262 IBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚60 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 122905600 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚61478912 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚322 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 5368709120 PCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚59049984 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚34735194 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚429 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚44038144 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚40899233 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 M@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚59049984 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 4654602240 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 2670649419776ÃÕîgf$×½O G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚262 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 5494538240 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 2670691364864 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚301427734872064 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17104896 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚301427734872064 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 4552982528 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚302312482578769 KBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚4876 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚209 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚299647412240384 KDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚60 @7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚5264 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚9272529 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚41945088 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚304098367186944 UHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚64681984 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4289 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚20643840 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 4654602240 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚265591845862400 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚322 _step‚908 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚4287 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚40899233 G>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4876 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚41945088 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚299647395201024 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 5368709120 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 4517085184 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚223 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚304098426236928 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚9272587 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚262 IBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚60 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 122905600 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚61478912 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚322 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 5368709120 PCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚59049984 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚34735194 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚429 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚44038144 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚40899233 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 M@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚59049984 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 4654602240 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 2670649419776 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 4552982528 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚265591910544384 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 2824164785152 trainer/global_step‚11049 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚37088891 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚37088891 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 5368709120 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 D;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚5264 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 2665128725765 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 125829120 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚301427717767168 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚262767701721088 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚302312540966149 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚37084604 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 2670649419776 IDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚262767681077248 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚304098367186944 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚41348020 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 122905600 PIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚58 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 2824208823296 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 2665087377745 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 125829120 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚262 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17104896 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚37084604 IDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚301454040104960 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚302338957332345 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 125829120 K>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59053056 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚20643840 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚77994970 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 122905600 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 4552982528 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 2670922436608 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚34738287 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 4552982528 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚61478912 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 NGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚60 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚9273355 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 2665359298425 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚322 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚40902786 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 845178880 NADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚59053056 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚41948160 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚34738225 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚262790749093888 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚299673598033920 >5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5264 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654602240 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 _step‚909 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚304124979646464 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚77994970 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 trainer/global_step‚11049 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚37087891 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 4654602240 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚41948160 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚40902784 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚265615138954240 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚77990675 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚301454057209856 E8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 NADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚58387428 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521206 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚262 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚262 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚77990675 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚9273415 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 2824454539264 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚322 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚322 D6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚322 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 5494538240 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 2670880488448 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚265615203633152 SFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚64678912 B8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 NADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚58387428 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521206 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚262 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚262 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚77990675 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚9273415 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 2824454539264 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚322 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚322 D6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚322 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 5494538240 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 2670880488448 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚265615203633152 SFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚64678912 BDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚301454040104960 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚302338957332345 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 125829120 K>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59053056 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚20643840 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚77994970 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 122905600 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 4552982528 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 2670922436608 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚34738287 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 4552982528 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚61478912 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 NGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚60 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚9273355 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 2665359298425 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚322 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚40902786 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 845178880 NADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚59053056 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚41948160 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚34738225 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚262790749093888 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚299673598033920 >5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5264 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654602240 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 _step‚909 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚304124979646464 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚77994970 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 trainer/global_step‚11049 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚37087891 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 4654602240 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚41948160 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚40902784¢·\2Û- _wandbruntime‚2950  _runtime‚2950¡IÇrÜì 0loss/FRt1/assignment_loss‚0.5674884210281738 0loss/FRt1/detection_loss‚0.49290556966955623 #loss/total_loss‚2.1040015700546 epoch‚27 trainer/global_step‚11049 % _timestamp‚1.7866616019273713e+09 _runtime‚2950.457173741 _step‚910 /loss/FRt2/assignment_loss‚0.566754951880504 0loss/FRt2/detection_loss‚0.47685262747636636ùqð{wÝñ 0loss/FRt1/detection_loss‚0.49290556966955623 #loss/total_loss‚2.1040015700546 epoch‚27 trainer/global_step‚11049 % _timestamp‚1.7866616019273713e+09 _runtime‚2950.457173741 _step‚910 /loss/FRt2/assignment_loss‚0.566754951880504 0loss/FRt2/detection_loss‚0.47685262747636636 0loss/FRt1/assignment_loss‚0.5674884210281738ŽÆ7–Þ‚(:‹ æ•ùÓçºúà gpu.0.gpu‚64.0 gpu.0.memory‚12, gpu.0.memoryAllocated‚7.0313262939453125) gpu.0.memoryAllocatedBytes‚ 6039863296 gpu.0.temp‚34.0 gpu.0.powerWatts‚124.576' gpu.0.enforcedPowerLimitWatts‚500.0 gpu.0.powerPercent‚24.9152 gpu.0.smClock‚1410 gpu.0.memoryClock‚1593$ gpu.0.correctedMemoryErrors‚1955# gpu.0.uncorrectedMemoryErrors‚0º’KŽß‚(:ƒ æ•ùÓŸÖâ cpu‚27.17963023790554 disk.drbd2165.in‚0 disk.sda3.in‚26968.33203125# disk.nvme0c0n1.in‚ 43584.3828125% disk.nvme0c0n1.out‚57128.75390625 disk.sda.out‚ 54.9296875 disk.sda1.out‚0) proc.memory.availableMB‚ 5307.17578125 network.recv‚ 5.532633e+06* disk./.usagePercent‚32.725376795985014 disk.sda3.out‚ 54.9296875 disk.sda.in‚26968.33203125 proc.cpu.threads‚17 network.sent‚ 1.4399903e+07% disk./.usageGB‚487.63176345825195 disk.sda2.out‚0! disk.nvme0n1.in‚ 43584.3828125 disk.dm-1.out‚ 54.9296875 disk.dm-0.out‚57128.75390625 disk.sda1.in‚0) proc.memory.percent‚14.17388916015625 disk.sda2.in‚0# disk.nvme0n1.out‚57128.75390625 disk.dm-1.in‚26968.33203125 disk.drbd2165.out‚0 disk.dm-0.in‚ 43584.3828125$ memory_percent‚67.60756969451904 proc.memory.rssMB‚2322.25Ž `å2à- _wandbruntime‚2962  _runtime‚2962˜}ÐÈá trainer/global_step‚11099 % _timestamp‚1.7866616140145023e+09 _runtime‚2962.544539753 _step‚911 & lr-AdamW/pg1‚0.0006423012208169413 & lr-AdamW/pg2‚0.0006423012208169413;ßÛÍâÇ & lr-AdamW/pg2‚0.0006423012208169413 trainer/global_step‚11099 % _timestamp‚1.7866616140145023e+09 _runtime‚2962.544539753 _step‚911 & lr-AdamW/pg1‚0.0006423012208169413ú~ù¯2ã- _wandbruntime‚2962  _runtime‚2962UÏSB$ä¸O GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚305426129892352 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚34886568 UHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚70973440 _runtime‚2962.548892399 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚302743851761664 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚61478912 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚303633392453557 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 5368709120 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚4287 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 122905600 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 122905600 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚37253993 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚25572915 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚262 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚429 F?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚60 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 2682236185600 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4289 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚303633334066177 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚262 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 M@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚59049984 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚302743851761664 IDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 G>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4876 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 2682236185600 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚4289 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚41076881 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚263920049717248 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 IBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚60 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚209 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚263920070361088 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 2836482778624 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚41076881 PCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚58387380 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚37249706 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 125829120 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚37253993 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚41945088 D;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚5264 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 125829120 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 E8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚78326587 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 4552982528 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚9313720 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚50329600 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚300956684877824 % _timestamp‚1.7866616140154393e+09 @7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚5264 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 4517085184 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 5494538240 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚302743834656768 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚41945088 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚302743834656768 KBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚4876 IBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚69 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 5494538240 KDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚60 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚305426070842368 trainer/global_step‚11099 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚41076883 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚322 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚41348020 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 4552982528 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚78330876 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚266756553139712 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚262 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚37249706 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚20643840 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚300956701917184 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚322 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 2682278130688 PCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚59049984 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 2682278130688 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 2676649188353 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 5368709120 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚4287 PIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚67 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 5494538240 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 122306526 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚305426129892352 JBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚665 BDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚305426070842368 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚34886637 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 125829120 _step‚912 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚266756482166272)ÚXÃ'å½O UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 5368709120 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚4287 PIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚67 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 5494538240 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 122306526 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚305426129892352 JBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚665 BDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚305426070842368 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚34886637 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 125829120 _step‚912 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚266756482166272 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚305426129892352 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚34886568 UHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚70973440 _runtime‚2962.548892399 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚302743851761664 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚61478912 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚303633392453557 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 5368709120 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚4287 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 122905600 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 122905600 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚37253993 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚25572915 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚262 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚429 F?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚60 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 2682236185600 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4289 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚303633334066177 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚262 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 M@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚59049984 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚302743851761664 IDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 G>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4876 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 2682236185600 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚4289 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚41076881 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚263920049717248 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 IBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚60 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚209 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚263920070361088 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 2836482778624 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚41076881 PCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚58387380 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚37249706 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 125829120 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚37253993 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚41945088 D;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚5264 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 125829120 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 E8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚78326587 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 4552982528 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚9313720 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚50329600 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚300956684877824 % _timestamp‚1.7866616140154393e+09 @7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚5264 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 4517085184 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 5494538240 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚302743834656768 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚41945088 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚302743834656768 KBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚4876 IBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚69 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 5494538240 KDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚60 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚305426070842368 trainer/global_step‚11099 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚41076883 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚322 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚41348020 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 4552982528 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚78330876 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚266756553139712 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚262 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚37249706 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚20643840 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚300956701917184 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚322 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 2682278130688 PCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚59049984 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 2682278130688 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 2676649188353fçRp2æ- _wandbruntime‚2962  _runtime‚2962hc¿Ç&çÁM UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚41948160 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚37257286 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚41948160 SFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚66776064 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚4295 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚302770174099456 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚37252993 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚266779842230784 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 NADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚58387428 G@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚60 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 4552982528 BDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚305452683301888 E6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚322 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚41080436 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚305452624248832 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 122905600 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚303659808819753 _runtime‚2962.707157037 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚303659750432325 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚322 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 125829120 H@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚665 NGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚61 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 845178880 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 5494538240 D=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚60 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚302770156994560 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚41080434 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚41080434 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚78337722 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚78337722 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚41080436 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚9314519 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚300982870671360 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚41348068 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚262 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚223 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521206 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 4517085184 >5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5264 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚266779775454720 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚61478912 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚305452683301888 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚78333427 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚263943117733888 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 2836724496896 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 5494538240 trainer/global_step‚11099 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚37257286 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 2682467254272 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚302770156994560 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚25575120 B9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚5264 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚20643840 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚322 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚262 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 5368709120 IBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚60 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚4293 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚37252993 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17104896 G@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚63 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚34889578 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 2676879760965 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 122905600 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 2836678364672 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 5494538240 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 5368709120 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 125829120 K>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59053056 I@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚4876 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚34889641 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 831258624 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 2676921109033 $ _timestamp‚1.786661614176789e+09 EDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654602240 _step‚913 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚209 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 DDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0‘û©DÌ&èÆM RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 125829120 H@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚665 NGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚61 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 845178880 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 5494538240 D=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚60 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚302770156994560 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚41080434 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚41080434 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚78337722 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚78337722 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚41080436 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚9314519 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚300982870671360 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚41348068 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚262 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚223 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521206 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 4517085184 >5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5264 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚266779775454720 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚61478912 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚305452683301888 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚78333427 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚263943117733888 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 2836724496896 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 5494538240 trainer/global_step‚11099 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚37257286 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 2682467254272 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚302770156994560 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚25575120 B9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚5264 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚20643840 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚322 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚262 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 5368709120 IBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚60 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚4293 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚37252993 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17104896 G@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚63 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚34889578 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 2676879760965 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 122905600 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 2836678364672 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 5494538240 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 5368709120 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 125829120 K>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59053056 I@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚4876 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚34889641 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 831258624 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 2676921109033 $ _timestamp‚1.786661614176789e+09 EDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654602240 _step‚913 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚209 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 DDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚41948160 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚37257286 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚41948160 SFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚66776064 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚4295 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚302770174099456 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚37252993 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚266779842230784 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 NADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚58387428 G@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚60 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 4552982528 BDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚305452683301888 E6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚322 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚41080436 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚305452624248832 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 122905600 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚303659808819753 _runtime‚2962.707157037 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚303659750432325 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚322‘ƒKßË2é- _wandbruntime‚2962  _runtime‚2962ÞFÞrêì epoch‚27 trainer/global_step‚11099 _runtime‚2962.707966866 _step‚914 /loss/FRt2/detection_loss‚0.5184548846623841 %loss/total_loss‚2.277086826053144 % _timestamp‚1.7866616141777847e+09 0loss/FRt1/assignment_loss‚0.6273290052445747 0loss/FRt2/assignment_loss‚0.6678884703825211 .loss/FRt1/detection_loss‚0.463414465763664•á2wëñ epoch‚27 trainer/global_step‚11099 _runtime‚2962.707966866 _step‚914 /loss/FRt2/detection_loss‚0.5184548846623841 %loss/total_loss‚2.277086826053144 % _timestamp‚1.7866616141777847e+09 0loss/FRt1/assignment_loss‚0.6273290052445747 0loss/FRt2/assignment_loss‚0.6678884703825211 .loss/FRt1/detection_loss‚0.463414465763664’Þœh#”ì‚(:‰ õ•ùÓûñ‡á gpu.0.gpu‚49.0 gpu.0.memory‚8, gpu.0.memoryAllocated‚7.0313262939453125) gpu.0.memoryAllocatedBytes‚ 6039863296 gpu.0.temp‚35.0 gpu.0.powerWatts‚220.595' gpu.0.enforcedPowerLimitWatts‚500.0 gpu.0.powerPercent‚44.119 gpu.0.smClock‚1410 gpu.0.memoryClock‚1593$ gpu.0.correctedMemoryErrors‚1955# gpu.0.uncorrectedMemoryErrors‚0!Q;í‚(:ô õ•ùÓ¿üˆä disk.sda2.out‚0 disk.dm-1.out‚55.25 disk.drbd2165.in‚0 disk.drbd2165.out‚0# disk.nvme0c0n1.in‚ 43584.3828125% disk.nvme0c0n1.out‚57133.71484375 disk.sda1.out‚0) proc.memory.availableMB‚ 5306.80078125% disk./.usageGB‚487.63175201416016 disk.sda3.out‚55.25 disk.sda.in‚26968.33203125 disk.sda1.in‚0 proc.memory.rssMB‚2322.25 cpu‚27.179373216128116 network.sent‚ 1.4472962e+07 network.recv‚ 5.563291e+06) disk./.usagePerceœUœÄxnt‚32.72537602796241 disk.sda2.in‚0! disk.nvme0n1.in‚ 43584.3828125 disk.dm-1.in‚26968.33203125 disk.dm-0.out‚57133.71484375$ memory_percent‚67.60985851287842# disk.nvme0n1.out‚57133.71484375 disk.dm-0.in‚ 43584.3828125 disk.sda3.in‚26968.33203125 disk.sda.out‚55.25) proc.memory.percent‚14.17388916015625 proc.cpu.threads‚17G4:2î- _wandbruntime‚2974  _runtime‚2974e®*úÇïÁ & lr-AdamW/pg2‚0.0006363563375172002 trainer/global_step‚11149 % _timestamp‚1.7866616262132733e+09 _runtime‚ 2974.74497104 _step‚915 & lr-AdamW/pg1‚0.0006363563375172002Jî+TÌðÆ & lr-AdamW/pg1‚0.0006363563375172002 & lr-AdamW/pg2‚0.0006363563375172002 trainer/global_step‚11149 % _timestamp‚1.7866616262132733e+09 _runtime‚ 2974.74497104 _step‚915“ßp2ñ- _wandbruntime‚2974  _runtime‚2974ÊDr¼'ò¶O @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚322 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚37414808 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17104896 D;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚5264 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚4287 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚306753774497792 F>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 5368709120 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚262 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚304059968651264 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚265072418357248 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 5494538240 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 M@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚59049984 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚306753774497792 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚41945088 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 2693822951424 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 122905600 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚41254533 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚262 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚262 IBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚70 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 IBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚60 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 2848717843968 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚302265974554624 % _timestamp‚1.7866616262139544e+09 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚37419095 PIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚68 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 2688252346981 E8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚78669339 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 4654602240 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4289 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚267921205077504 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚25683165 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚304954243940965 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 2688210998961 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚37419095 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 122306526 _runtime‚ 2974.7451037 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚304059951546368 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚37414808 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚265072439001088 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 125829120 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 2693864896512 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 2693864896512 UHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚68876288 @7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚5264 PCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚59049984 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚48232448 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚304954185553585 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚306753833547776 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 5494538240 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 4654602240 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚41945088 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚41254533 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17104896 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚61478912 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚304059968651264 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 122905600 IDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4876 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 845178880D¼!Á'ó»O F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 4654602240 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚41945088 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚41254533 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17104896 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚61478912 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚304059968651264 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 122905600 IDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4876 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 845178880 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚322 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚37414808 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17104896 D;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚5264 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚4287 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚306753774497792 F>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 5368709120 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚262 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚304059968651264 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚265072418357248 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 5494538240 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 M@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚59049984 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚306753774497792 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚41945088 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 2693822951424 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 122905600 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚41254533 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚262 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚262 IBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚70 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 IBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚60 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 2848717843968 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚302265974554624 % _timestamp‚1.7866616262139544e+09 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚37419095 PIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚68 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 2688252346981 E8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚78669339 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 4654602240 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4289 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚267921205077504 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚25683165 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚304954243940965 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 2688210998961 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚37419095 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 122306526 _runtime‚ 2974.7451037 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚304059951546368 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚37414808 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚265072439001088 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 125829120 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 2693864896512 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 2693864896512 UHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚68876288 @7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚5264 PCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚59049984 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚48232448 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚304954185553585 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚306753833547776 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 5494538240 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0”Cé­Æ2ô- _wandbruntime‚2974  _runtime‚2974ê-È&õÂM >5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5264 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚4293 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚265095465730048 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 5494538240 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚322 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚37418095 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚304086290989056 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚78676179 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚41258084 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 2688441571573 DDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654602240 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚61478912 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 122306526 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 D=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚60 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 2694095968256 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚306780386957312 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚78676179 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚306780386957312 G@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚59 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚9355189 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17104896 NADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚59053056 E6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚322 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚262 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 122905600 EDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚78680474 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 2688482919641 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚78680474 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚267944491022848 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 125829120 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚262 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 2694054020096 K>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59053056 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚4293 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚306780327904256 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 4552982528 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚304980601919733 P6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚322 G@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚60 _step‚917 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚37422388 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚41948160 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 I@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚4876 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚37743616 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚304980660307161 NADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚58387428 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚267944432635392 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚35040559 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚302292160348160 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 5368709120 _runtime‚2974.906142267 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 4654602240 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521206 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚302292177387520 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚41258086 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 BDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚429 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚304086273884160 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚9355246 IBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚60 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚41258086 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚304086273884160 B9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚5264 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚25685372 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17104896 NGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚57 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 2849004648960 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 125829120 SFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚58387456 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚265095486373888 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚20643840 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 5494538240 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 4517085184 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚41348068 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 5368709120 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚41258084 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚37418095 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚322 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 4552982528 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚35040618 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚25685370 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 831258624 trainer/global_step‚11149 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 125829120MIêÉÔöÇM UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17104896 NADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚59053056 E6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚322 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚262 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 122905600 EDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚78680474 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 2688482919641 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚78680474 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚267944491022848 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 125829120 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚262 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 2694054020096 K>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59053056 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚4293 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚306780327904256 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 4552982528 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚304980601919733 P6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚322 G@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚60 _step‚917 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚37422388 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚41948160 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 I@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚4876 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚37743616 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚304980660307161 NADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚58387428 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚267944432635392 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚35040559 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚302292160348160 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 5368709120 _runtime‚2974.906142267 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 4654602240 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521206 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚302292177387520 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚41258086 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 BDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚429 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚304086273884160 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚9355246 IBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚60 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚41258086 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚304086273884160 B9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚5264 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚25685372 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17104896 NGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚57 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 2849004648960 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 125829120 SFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚58387456 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚265095486373888 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚20643840 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 5494538240 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 4517085184 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚41348068 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 5368709120 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚41258084 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚37418095 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚322 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 4552982528 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚35040618 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚25685370 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 831258624 trainer/global_step‚11149 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 125829120 >5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5264 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚4293 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚265095465730048 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 5494538240 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚322 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚37418095 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚304086290989056 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚78676179 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚41258084 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 2688441571573 DDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654602240 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚61478912 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 122306526 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 D=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚60 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 2694095968256 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚306780386957312 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚78676179 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚306780386957312 G@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚59 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚9355189•7ž4Œ2÷- _wandbruntime‚2974  _runtime‚2974Y(üÃsøí 0loss/FRt1/detection_loss‚0.49029975549408145 /loss/FRt2/detection_loss‚0.5180177413364175 epoch‚27 % _timestamp‚1.7866616263767931e+09 _step‚918 /loss/FRt1/assignment_loss‚0.579397130207129 0loss/FRt2/assignment_loss‚0.6081824470078958 %loss/total_loss‚2.195897074045524 trainer/global_step‚11149 _runtime‚2974.906609127*FÂøxùò 0loss/FRt1/detection_loss‚0.49029975549408145 /loss/FRt2/detection_loss‚0.5180177413364175 epoch‚27 % _timestamp‚1.7866616263767931e+09 _step‚918 /loss/FRt1/assignment_loss‚0.579397130207129 0loss/FRt2/assignment_loss‚0.6081824470078958 %loss/total_loss‚2.195897074045524 trainer/global_step‚11149 _runtime‚2974.906609127–Â'¿5•ú‚(:Š „–ùÓÇÅñà gpu.0.gpu‚64.0 gpu.0.memory‚12, gpu.0.memoryAllocated‚7.0313262939453125) gpu.0.memoryAllocatedBytes‚ 6039863296 gpu.0.temp‚35.0 gpu.0.powerWatts‚166.715' gpu.0.enforcedPowerLimitWatts‚500.0 gpu.0.powerPercent‚33.343 gpu.0.smClock‚1410 gpu.0.memoryClock‚1593$ gpu.0.correctedMemoryErrors‚1955# gpu.0.uncorrectedMemoryErrors‚0‰æÞÜ‘û‚(:† „–ùÓöÍÐâ network.sent‚ 1.4536281e+07( disk./.usagePercent‚32.7253811481131 disk.dm-1.in‚27019.05859375# disk.dm-0.out‚57140.10498046875 disk.sda3.out‚ 55.359375 disk.sda.in‚27019.05859375 disk.sda1.in‚0 proc.cpu.threads‚17 network.recv‚ 5.582837e+06! disk.nvme0n1.in‚ 43596.4609375 disk.sda2.out‚0 cpu‚27.17933685013032% disk./.usageGB‚487.63182830810547& disk.nvme0n1.out‚57140.10498046875 disk.dm-1.out‚ 55.359375 disk.dm-0.in‚ 43596.4609375( disk.nvme0c0n1.out‚57140.10498046875 disk.sda1.out‚0$ memory_percent‚67.98548698425293( proc.memory.availableMB‚ 5245.2578125 disk.drbd2165.in‚0 disk.drbd2165.out‚0 disk.sda3.in‚27019.05859375# disk.nvme0c0n1.in‚ 43596.4609375 disk.sda.out‚ 55.359375 disk.sda2.in‚0 proc.memory.rssMB‚2322.25) proc.memory.percent‚14.17388916015625Ì gš2ü- _wandbruntime‚2987  _runtime‚2987xçú(Èý _runtime‚2987.006835021 _step‚919 & lr-AdamW/pg1‚0.0006304187533375402 & lr-AdamW/pg2‚0.0006304187533375402 trainer/global_step‚11199 % _timestamp‚1.7866616384767082e+09Hø ÍþÇ _runtime‚2987.006835021 _step‚919 & lr-AdamW/pg1‚0.0006304187533375402 & lr-AdamW/pg2‚0.0006304187533375402 trainer/global_step‚11199 % _timestamp‚1.7866616384767082e+09—¸|þÐ2ÿ- _wandbruntime‚2987  _runtime‚2987×½Ù½'€·O SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 4552982528 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚37584197 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 5494538240 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 125829120 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 122905600 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 IDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚223 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚209 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 5494538240 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚322 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 4654602240 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 122306526 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚37579910 IDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4876 PCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚59049984 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 5368709120 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 4552982528 KBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚4876 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚322 KDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚60 _runtime‚2987.008111733 C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚4289 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚41432183 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚306275095428373 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚303575281270784 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 2705409717248 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚4287 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚41348020 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 @7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚5264 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚305376085540864 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 4654602240 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 IBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚60 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚308081537203200 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17104896 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 831258624 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚305376068435968 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4289 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚266224786997248 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚61478912 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 UHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚66779136 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 5368709120 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚322 JBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚665 IBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚72 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 4517085184 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 E8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚79012091 M@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚59049984 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚25793417 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚305376085540864 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚37579910 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚25793415 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 5368709120 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚322 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚308081478153216 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 4628521206 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚305376068435968 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚308081537203200 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 125829120 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚4287 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚41945088 D;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚5264 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚269085805080576 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 2699772809569 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚308081478153216 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚46135296 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 2861018083328 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚262 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17104896 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚9395294 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚269085871859712€ßÃiÂ'¼O MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚262 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17104896 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚9395294 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚269085871859712 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 4552982528 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚37584197 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 5494538240 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 125829120 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 122905600 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 IDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚223 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚209 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 5494538240 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚322 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 4654602240 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 122306526 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚37579910 IDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4876 PCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚59049984 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 5368709120 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 4552982528 KBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚4876 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚322 KDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚60 _runtime‚2987.008111733 C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚4289 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚41432183 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚306275095428373 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚303575281270784 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 2705409717248 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚4287 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚41348020 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 @7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚5264 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚305376085540864 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 4654602240 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 IBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚60 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚308081537203200 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17104896 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 831258624 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚305376068435968 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4289 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚266224786997248 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚61478912 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 UHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚66779136 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 5368709120 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚322 JBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚665 IBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚72 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 4517085184 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 E8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚79012091 M@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚59049984 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚25793417 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚305376085540864 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚37579910 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚25793415 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 5368709120 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚322 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚308081478153216 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 4628521206 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚305376068435968 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚308081537203200 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 125829120 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚4287 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚41945088 D;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚5264 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚269085805080576 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 2699772809569 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚308081478153216 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚46135296 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 2861018083328˜}Þî2‚- _wandbruntime‚2987  _runtime‚2987ÆÜ€¸ƒÂM WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚61478912 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚322 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 DDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚37583197 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚306301453407141 IBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocmñÞ"ated‚60 _runtime‚2987.174644508 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 5494538240 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚308108031559680 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚79018931 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17104896 NGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚74 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 845178880 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 2861259681792 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 2705682734080 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚9396124 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 5368709120 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚262 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚79018931 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚35191746 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚269109165022208 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚50326528 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 2700003382181 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 125829120 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 2705640785920 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚79023226 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521206 G@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚60 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 4654602240 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 831258624 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 2700044730249 _step‚921 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚305402390773760 EDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654602240 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚303601467064320 % _timestamp‚1.7866616386443715e+09 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚37583197 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚37587490 B9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚5264 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 5368709120 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 D=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚60 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚308108090612736 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 122905600 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚25795620 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 2861310008320 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 122306526 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 5368709120 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚41435734 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚266247855013888 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 4552982528 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚41435734 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚35191670 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 4517085184 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚262 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚209 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚20643840 PDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 K>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59053056 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚37587490 I@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚4876 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚41348068 trainer/global_step‚11199 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 2705682734080 H@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚665 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚223 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚303601450024960 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚322 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚41435736 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 NADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚58387428 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 125829120 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚322 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚322 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚262 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚79023226 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚4295 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚41948160 SFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚70970368 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚266247834370048 B5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5264 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚305402407878656 NADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚59053056 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚308108090612736 G@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚76 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚305402407878656ñôÅÍ&„ÇM GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 2705640785920 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚25795622 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 4552982528 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 5494538240 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚4293 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚269109094051840 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 K>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59053056 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚37587490 I@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚4876 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚41348068 trainer/global_step‚11199 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 2705682734080 H@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚665 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚223 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚303601450024960 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚322 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚41435736 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 NADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚58387428 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 125829120 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚322 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚322 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚262 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚79023226 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚4295 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚41948160 SFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚70970368 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚266247834370048 B5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5264 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚305402407878656 NADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚59053056 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚308108090612736 G@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚76 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚305402407878656 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚61478912 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚322 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 DDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚37583197 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚306301453407141 IBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚60 _runtime‚2987.174644508 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 5494538240 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚308108031559680 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚79018931 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17104896 NGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚74 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 845178880 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 2861259681792 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 2705682734080 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚9396124 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 5368709120 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚262 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚79018931 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚35191746 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚269109165022208 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚50326528 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 2700003382181 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 125829120 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 2705640785920 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚79023226 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521206 G@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚60 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 4654602240 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 831258624 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 2700044730249 _step‚921 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚305402390773760 EDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654602240 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚303601467064320 % _timestamp‚1.7866616386443715e+09 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚37583197 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚37587490 B9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚5264 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 5368709120 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 D=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚60 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚308108090612736 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 122905600 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚25795620 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 2861310008320 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 122306526 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 5368709120 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚41435734 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚266247855013888 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 4552982528 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚41435734 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚35191670 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 4517085184 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚262 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚209 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚20643840 PDeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4289 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚9444997 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚307980912379722 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚322 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚25944279 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 2877439191552 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚262 KDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚60 _runtime‚3004.557891383 G>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4876 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 4654602240 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 2721064206848 F?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚60 % _timestamp‚1.7866616560240254e+09 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚9445070 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 831258624 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 5494538240 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17104896 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚20643840 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 M@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚59049984 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 4552982528 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 4654602240 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚307083139645440 JBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚665 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 4517085184 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 2721106151936 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚37771486 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 845178880 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 4552982528 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚41945088 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 2715400592278 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚307083122540544 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚209 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚267676446588928 BDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚262 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚41670437 F>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 2715441940298 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 IBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚60 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚223 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚309804186747392 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17104896 IDeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4289 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚9444997 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚307980912379722 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚322 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚25944279 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 2877439191552 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚262 KDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚60 _runtime‚3004.557891383 G>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4876 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 4654602240 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 2721064206848 F?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚60 % _timestamp‚1.7866616560240254e+09 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚9445070 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 831258624 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 5494538240 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17104896 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚20643840 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 M@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚59049984 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 4552982528 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 4654602240 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚307083139645440 JBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚665 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 4517085184 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 2721106151936 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚37771486 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 845178880 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 4552982528 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚41945088 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 2715400592278 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚307083122540544 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚209 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚267676446588928 BDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚262 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚41670437 F>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 2715441940298 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 IBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚60 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚223 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚309804186747392 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17104896 IDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚35392314 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚209 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 _runtime‚3004.719265727 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚79444476 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17104896 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚309830799206912 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚41948160 EDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59053056 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚270577107571200 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 122905600 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 2877634253312 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 G@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚60 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 122905600 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 4517085184 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 122306526 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 125829120 EDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654602240 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚79448771 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚41673990 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚4293 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚262 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚41673992 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17104896 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 2721337223680 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚35392378 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 125829120 D=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚60 % _timestamp‚1.7866616561888976e+09 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚44035072 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 B8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚37774779 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚270577172250112 B9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚5264 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚9445830 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 5494538240 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 DDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 5494538240 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚307109461983232 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 2721295275520 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚25946484 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚305291639193600 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚322 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 4552982528 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚25946486 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚223 >5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5264 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚322 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚322 PDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚41673990 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚9445892 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521206 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚322 _step‚926 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚79448771 NADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚59053056 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 5368709120 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚305291656232960 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚41673992 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 125829120;wÌÍ&•ÇM PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 122905600 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 2877634253312 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 G@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚60 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 122905600 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 4517085184 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 122306526 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 125829120 EDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654602240 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚79448771 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚41673990 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚4293 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚262 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚41673992 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17104896 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 2721337223680 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚35392378 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 125829120 D=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚60 % _timestamp‚1.7866616561888976e+09 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚44035072 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 B8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚37774779 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚270577172250112 B9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚5264 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚9445830 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 5494538240 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 DDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 5494538240 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚307109461983232 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 2721295275520 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚25946484 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚305291639193600 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚322 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 4552982528 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚25946486 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚223 >5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5264 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚322 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚322 PDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚41673990 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚9445892 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521206 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚322 _step‚926 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚79448771 NADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚59053056 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 5368709120 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚305291656232960 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚41673992 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 125829120 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚41948160 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 5494538240 I@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚4876 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚35392314 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚209 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 _runtime‚3004.719265727 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚79444476 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17104896 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚309830799206912 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚41948160 EDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59053056 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚270577107571200ž|l2–- _wandbruntime‚3004  _runtime‚3004Ýå(Ls—í &loss/total_loss‚2.2644670647596117 % _timestamp‚1.7866616561899538e+09 /loss/FRt2/detection_loss‚0.5219796179942321 epoch‚28 trainer/global_step‚11249 _runtime‚3004.719769478 _step‚927 /loss/FRt1/assignment_loss‚0.680153165618173 0loss/FRt2/assignment_loss‚0.5829023170958721 /loss/FRt1/detection_loss‚0.4794319640513347‡ˆ†x˜ò % _timestamp‚1.7866616561899538e+09 /loss/FRt2/detection_loss‚0.5219796179942321 epoch‚28 trainer/global_step‚11249 _runtime‚3004.719769478 _step‚927 /loss/FRt1/assignment_loss‚0.680153165618173 0loss/FRt2/assignment_loss‚0.5829023170958721 /loss/FRt1/detection_loss‚0.4794319640513347 &loss/total_loss‚2.2644670647596117Ÿ8àU5•™‚(:Š ¢–ùÓ᳆á gpu.0.gpu‚64.0 gpu.0.memory‚12, gpu.0.memoryAllocated‚7.0313262939453125) gpu.0.memoryAllocatedBytes‚ 6039863296 gpu.0.temp‚36.0 gpu.0.powerWatts‚213.635' gpu.0.enforcedPowerLimitWatts‚500.0 gpu.0.powerPercent‚42.727 gpu.0.smClock‚1410 gpu.0.memoryClock‚1593$ gpu.0.correctedMemoryErrors‚1955# gpu.0.uncorrectedMemoryErrors‚0Yè{Cš‚(:„ ¢–ùÓç“åâ disk.sda1.out‚0 disk.sda2.in‚0ñ^¡àL cpu‚27.186253659918776 proc.cpu.threads‚17 network.sent‚ 1.4657315e+07* disk./.usagePercent‚32.725517856136385# disk.dm-0.out‚57151.15478515625 disk.sda3.out‚ 56.14453125 disk.sda.in‚ 28133.140625& disk.nvme0n1.out‚57151.15478515625 disk.drbd2165.out‚0) proc.memory.percent‚14.17388916015625 network.recv‚ 5.615691e+06 disk.dm-0.in‚ 43646.609375 disk.sda3.in‚ 28133.140625" disk.nvme0c0n1.in‚ 43646.609375( disk.nvme0c0n1.out‚57151.15478515625 disk.sda2.out‚0 disk.dm-1.out‚ 56.14453125( proc.memory.availableMB‚ 5244.5390625$ disk./.usageGB‚487.6338653564453 disk.sda1.in‚0 disk.nvme0n1.in‚ 43646.609375 disk.dm-1.in‚ 28133.140625 disk.drbd2165.in‚0# memory_percent‚67.9898738861084 proc.memory.rssMB‚2322.25 disk.sda.out‚ 56.14453125è-!£2›- _wandbruntime‚3016  _runtime‚3016(Et÷ÇœÁ trainer/global_step‚11299 $ _timestamp‚1.786661668168176e+09 _runtime‚3016.698237224 _step‚928 & lr-AdamW/pg1‚0.0006185670072849057 & lr-AdamW/pg2‚0.0006185670072849057Ý®ù4ÌÆ & lr-AdamW/pg1‚0.0006185670072849057 & lr-AdamW/pg2‚0.0006185670072849057 trainer/global_step‚11299 $ _timestamp‚1.786661668168176e+09 _runtime‚3016.698237224 _step‚928 t´Š|2ž- _wandbruntime‚3016  _runtime‚3016 F ½'Ÿ·O @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚322 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 122905600 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚37932301 KBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚4876 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚268828815228928 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 5368709120 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 2732650972672 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚41848087 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚322 IDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4876 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚311131890402816 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚429 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚26054529 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 845178880 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚311131949452800 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 4654602240 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 5368709120 @7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚5264 IDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 IBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚60 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚41848089 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 5494538240 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚311131890402816 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚9486313 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 5368709120 KDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚60 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 2732650972672 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚308399256535040 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚35540844 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 4517085184 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 2727003750906 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 2889712978944 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 125829120 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚61478912 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚262 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚262 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚41848087 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚308399239430144 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 4552982528 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚4287 UHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚68876288 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 5494538240 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 122905600 JBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚665 PCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚58387380 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 5494538240 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17104896 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚79784677 IBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚85 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚268828794585088 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 122306526 PCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚59049984 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4289 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 2726962402886 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 125829120 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚322 _runtime‚3016.702921265 F>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 4552982528 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚35540759 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚308399256535040 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 PIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚83 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚48232448 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚309301705479750 trainer/global_step‚11299 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚37932301 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17104896 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚308399239430144 BDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4876 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚311131890402816 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚429 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚26054529 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 845178880 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚311131949452800 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 4654602240 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 5368709120 @7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚5264 IDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 IBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚60 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚41848089 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 5494538240 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚311131890402816 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚9486313 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 5368709120 KDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚60 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 2732650972672 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚308399256535040 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚35540844 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 4517085184 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 2727003750906 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 2889712978944 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 125829120 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚61478912 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚262 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚262 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚41848087 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚308399239430144 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 4552982528 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚4287 UHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚68876288 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 5494538240 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 122905600 JBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚665 PCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚58387380 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 5494538240 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17104896 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚79784677 IBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚85 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚268828794585088 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 122306526 PCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚59049984 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4289 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 2726962402886 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 125829120 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚322 _runtime‚3016.702921265 F>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 4552982528 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚35540759 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚308399256535040 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 PIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚83 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚48232448 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚309301705479750 trainer/global_step‚11299 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚37932301 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17104896 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚308399239430144 BDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654602240 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚79791523 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521206 >5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5264 P8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 NADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚58387428 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 5494538240 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚311158502862336 G@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚76 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 2727192975498 G@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚60 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚311158502862336 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚41851642 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚37935588 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚271741755208704 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚262 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚308425578872832 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚41948160 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚79787228 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 5368709120 D=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚60 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚223 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚26056734 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 5494538240 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚306600945909760 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 122306526 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 125829120 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚262 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17104896 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚35543858 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚44035072 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 4517085184 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 5494538240 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚262 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚79787228 NGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚74 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚20643840 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 EDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 trainer/global_step‚11299 D6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚322 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚4295 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 2732923989504 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚308425578872832 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚9487048 SFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚64678912 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚308425561767936 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 2732923989504 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚26056736 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 EDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚35543782 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚322 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚322 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 2732882041344 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚271741819887616 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 845178880 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 831258624 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 5368709120 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 125829120 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚4293 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚268851862601728 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚268851841957888 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 BDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 B9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚5264 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚4293 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚322 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚41348068 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚41851640 K>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59053056 I@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚4876 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚311158443809280 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 4552982528 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚37939881 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚306600928870400 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚41851642 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚429 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚61478912 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 2727234323566©ü‘Cÿ£ÆM ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚306600928870400 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚41851642 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚429 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚61478912 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 2727234323566 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚308425561767936 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 5368709120 NADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚59053056 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654602240 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚79791523 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521206 >5DeviceStatsMonitor.on_train_batch_ÓéKÍ"end/active.all.peak‚5264 P8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 NADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚58387428 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 5494538240 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚311158502862336 G@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚76 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 2727192975498 G@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚60 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚311158502862336 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚41851642 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚37935588 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚271741755208704 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚262 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚308425578872832 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚41948160 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚79787228 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 5368709120 D=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚60 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚223 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚26056734 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 5494538240 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚306600945909760 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 122306526 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 125829120 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚262 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17104896 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚35543858 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚44035072 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 4517085184 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 5494538240 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚262 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚79787228 NGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚74 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚20643840 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 EDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 trainer/global_step‚11299 D6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚322 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚4295 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 2732923989504 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚308425578872832 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚9487048 SFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚64678912 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚308425561767936 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 2732923989504 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚26056736 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 EDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚35543782 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚322 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚322 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 2732882041344 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚271741819887616 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 845178880 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 831258624 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 5368709120 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 125829120 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚4293 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚268851862601728 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚268851841957888 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 BDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 B9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚5264 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚4293 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚322 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚41348068 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚41851640 K>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59053056 I@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚4876 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚311158443809280 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 4552982528 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚37939881¢Ëãj2¤- _wandbruntime‚3016  _runtime‚3016ˆ†›t¥î /loss/FRt1/assignment_loss‚0.619624513555897 0loss/FRt2/assignment_loss‚0.4990930669210485 /loss/FRt1/detection_loss‚0.5124124297323658 epoch‚28 trainer/global_step‚11299 0loss/FRt2/detection_loss‚0.48960699009952313 &loss/total_loss‚2.1207370003088344 % _timestamp‚1.7866616683329875e+09 _runtime‚3016.862830559 _step‚931tVÍhy¦ó epoch‚28 trainer/global_step‚11299 0loss/FRt2/detection_loss‚0.48960699009952313 &loss/total_loss‚2.1207370003088344 % _timestamp‚1.7866616683329875e+09 _runtime‚3016.862830559 _step‚931 /loss/FRt1/assignment_loss‚0.619624513555897 0loss/FRt2/assignment_loss‚0.4990930669210485 /loss/FRt1/detection_loss‚0.5124124297323658£ÿ#÷Ž2§- _wandbruntime‚3028  _runtime‚3028–UȨ & lr-AdamW/pg1‚0.0006126536066274336 & lr-AdamW/pg2‚0.0006126536066274336 trainer/global_step‚11349 % _timestamp‚1.7866616803891044e+09 _runtime‚3028.919124308 _step‚932“=®»Í©Ç & lr-AdamW/pg1‚0.0006126536066274336 & lr-AdamW/pg2‚0.0006126536066274336 trainer/global_step‚11349 % _timestamp‚1.7866616803891044e+09 _runtime‚3028.919124308 _step‚932¤}×»ð2ª- _wandbruntime‚3028  _runtime‚3028ÐN5̾'«¸O IDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚312459594058240 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 2744237738496 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚38097403 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚26164781 UHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚62584832 IDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚312459594058240 IBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚70 BDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚322 IBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚60 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚41945088 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚80127429 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚38101690 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚269981183868928 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚309715356319744 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚20643840 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 _runtime‚3028.923881859 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 4517085184 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚42025739 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚4287 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚42025737 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚41940992 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚262 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚312459653108224 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17104896 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 G>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4876 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚322 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 5368709120 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 125829120 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚35692201 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 2901921478144 _step‚933 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 2744279683584 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚272883147288064 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 125829120 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚322 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 5494538240 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4289 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚26164779 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚223 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚9527352 PCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚58387380 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 2744279683584 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚9527420 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚262 @7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚5264 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚429 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 2738565561514(Ð 4Ã'¬½O UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚310622556967158 % _timestamp‚1.7866616803900185e+09 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 4654602240 trainer/global_step‚11349 M@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚59049984 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17104896 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 831258624 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚307884049793024 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 4552982528 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚307884032753664 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 5368709120 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚38097403 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚309715373424640 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚272883084703232 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 4552982528 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 845178880 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚322 IBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚60 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚41945088 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚80127429 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚38101690 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚269981183868928 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚309715356319744 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚20643840 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 _runtime‚3028.923881859 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 4517085184 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚42025739 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚4287 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚42025737 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚41940992 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚262 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚312459653108224 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17104896 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 G>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4876 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚322 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 5368709120 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 125829120 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚35692201 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 2901921478144 _step‚933 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 2744279683584 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚272883147288064 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 125829120 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚322 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 5494538240 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4289 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚26164779 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚223 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚9527352 PCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚58387380 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 2744279683584 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚9527420 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚262 @7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚5264 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚429 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 2738565561514 IDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚312459594058240 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 2744237738496 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚38097403 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚26164781 UHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚62584832 IDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚312459594058240 IBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚70 BDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚35695138 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚61478912 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚310649031720734 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚322 trainer/global_step‚11349 J;DeviceStatsMonitor.on_‰#”´train_batch_end/active_bytes.all.peak‚ 4654602240 B9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚5264 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚35695209 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚262 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚80134275 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 2744510755328 H@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚665 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚9528154 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 4517085184 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚44035072 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 2738796134174 NADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚59053056 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 G@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚71 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 2744468807168 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 122905600 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚41948160 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 2744468807168 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 2902208250368 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 5368709120 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 125829120 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚262 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 831258624 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 _runtime‚3029.081551026 >5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5264 DDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59053056 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚312486147464704 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚272906439492096 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 845178880 BDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654602240 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17104896 NADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚58387428 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚307910218547200 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚4293 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚42029290 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 4552982528 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚26166984 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚312486206517760 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚312486206517760 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚309741695762432 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚223 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚307910235586560 IBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚60 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 122905600 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚80134275 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚80129980 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚262 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 D=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚60 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚80129980 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚309741678657536 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 2744510755328 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚429 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚26166986 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚9528223 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚270004231241728 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 2738754786106 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521206 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚41348068 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚4295 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚42029292 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚4293 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 4552982528 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚38104983 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚322 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17104896 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚322 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚310648973333306 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 125829120 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚38104983 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 SFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚64678912 % _timestamp‚1.7866616805512853e+09 NGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚69 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚272906374813184 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 125829120 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚309741695762432 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚38100690+¦„Í&¯ÇM A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚322 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17104896 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚322 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚310648973333306 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 125829120 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚38104983 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 SFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚64678912 % _timestamp‚1.7866616805512853e+09 NGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚69 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚272906374813184 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 125829120 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚309741695762432 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚38100690 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚38100690 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚209 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚270004210597888 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 2902164215296 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚41948160 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 5494538240 PDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚35695138 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚61478912 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚310649031720734 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚322 trainer/global_step‚11349 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 4654602240 B9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚5264 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚35695209 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚262 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚80134275 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 2744510755328 H@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚665 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚9528154 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 4517085184 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚44035072 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 2738796134174 NADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚59053056 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 G@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚71 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 2744468807168 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 122905600 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚41948160 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 2744468807168 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 2902208250368 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 5368709120 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 125829120 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚262 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 831258624 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 _runtime‚3029.081551026 >5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5264 DDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59053056 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚312486147464704 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚272906439492096 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 845178880 BDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654602240 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17104896 NADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚58387428 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚307910218547200 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚4293 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚42029290 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 4552982528 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚26166984 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚312486206517760 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚312486206517760 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚309741695762432 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚223 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚307910235586560 IBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚60 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 122905600 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚80134275 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚80129980 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚262 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 D=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚60 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚80129980 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚309741678657536 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 2744510755328 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚429 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚26166986 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚9528223 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚270004231241728 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 2738754786106 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521206 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚41348068 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚4295 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚42029292 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚4293 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 4552982528 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚38104983¦©5B2°- _wandbruntime‚3029  _runtime‚3029³å¸t±î % _timestamp‚1.7866616805523307e+09 _step‚935 1loss/FRt2/assignment_loss‚0.48129316775146896 0loss/FRt1/detection_loss‚0.48000665703359696 /loss/FRt2/detection_loss‚0.4767230448570661 $loss/total_loss‚2.03178458916639 _runtime‚3029.082467959 0loss/FRt1/assignment_loss‚0.5937617195242577 epoch‚28 trainer/global_step‚11349•­¾y²ó 0loss/FRt1/assignment_loss‚0.5937617195242577 epoch‚28 trainer/global_step‚11349 % _timestamp‚1.7866616805523307e+09 _step‚935 1loss/FRt2/assignment_loss‚0.48129316775146896 0loss/FRt1/detection_loss‚0.48000665703359696 /loss/FRt2/detection_loss‚0.4767230448570661 $loss/total_loss‚2.03178458916639 _runtime‚3029.082467959§W6g“³‚(:ˆ ±–ùÓÔ×óà gpu.0.gpu‚0.0 gpu.0.memory‚0, gpu.0.memoryAllocated‚7.0313262939453125) gpu.0.memoryAllocatedBytes‚ 6039863296 gpu.0.temp‚34.0 gpu.0.powerWatts‚165.745' gpu.0.enforcedPowerLimitWatts‚500.0 gpu.0.powerPercent‚33.149 gpu.0.smClock‚1410 gpu.0.memoryClock‚1593$ gpu.0.correctedMemoryErrors‚1955# gpu.0.uncorrectedMemoryErrors‚0-»Aɉ´‚(:þ ±–ùÓ¦°Õâ) proc.memory.availableMB‚ 5244.70703125) proc.memory.percent‚14.17388916015625 network.sent‚ 1.4722428e+07% disk./.usageGB‚488.13603591918945 disk.sda3.in‚ 28133.140625 disk.sda.out‚ 56.3203125$ memory_percent‚67.98884868621826 proc.memory.rssMB‚2322.25 cpu‚27.18591564002933 disk.dm-1.in‚ 28133.140625 disk.drbd2165.in‚0" disk.dm-0.out‚57156.7353515625 disk.sda1.in‚0 proc.cpu.threads‚17 network.recv‚ 5.63814e+06 disk.dm-0.in‚ 43646.609375 disk.sda3.out‚ 56.3203125" disk.nvme0c0n1.in‚ 43646.609375' disk.nvme0c0n1.out‚57156.7353515625 disk.sda.in‚ 28133.140625 disk.sda2.in‚0 disk.sda2.out‚0) disk./.usagePercent‚32.75921894395131 disk.dm-1.out‚ 56.3203125 disk.drbd2165.out‚0 disk.sda1.out‚0 disk.nvme0n1.in‚ 43646.609375% disk.nvme0n1.out‚57156.7353515625õPŽò2µ- _wandbruntime‚3041  _runtime‚3041È~Kȶ _step‚936 & lr-AdamW/pg1‚0.0006067490274721548 & lr-AdamW/pg2‚0.0006067490274721548 trainer/global_step‚11399 % _timestamp‚1.7866616927497406e+09 _runtime‚3041.279767871À¶­Í·Ç & lr-AdamW/pg1‚0.0006067490274721548 & lr-AdamW/pg2‚0.0006067490274721548 trainer/global_step‚11399 % _timestamp‚1.7866616927497406e+09 _runtime‚3041.279767871 _step‚936¨w¤ÂŒ2¸- _wandbruntime‚3041  _runtime‚3041Ð<ä¾'¹¸O @7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚5264 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚223 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚313787356763648 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚35843226 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 2914233396736 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚311031473209344 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 4552982528 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚274047723320832 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚311943466841946 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚322 _runtime‚3041.283872083 G>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4876 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚311031473209344 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚309193322430464 D;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚5264 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 2914191455744 PCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚58387380 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 5368709120 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚322 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚42203387 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 122306526 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚311031490314240 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚271133531865088 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 trainer/global_step‚11399 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚61478912 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚4287 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 2755866449408 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 2755824504320 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚322 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚41348020 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 5494538240 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 PCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚59049984 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚20643840 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 831258624 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 2750086024102 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 F?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚60 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚9568138 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚38266792 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 4552982528 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚9568195 UHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚62584832 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚322 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 4654602240 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17104896 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 122905600 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 845178880 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 5368709120 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚42203389 E8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚80465892 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17104896 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚274047785905664 BDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚313787297713664 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚38262505 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚209 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 KDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚60 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚311943408454566 IBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚59 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚429 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 2755824504320 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 4654602240 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚41945088 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚271133552508928 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4289 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚41940992 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 4517085184 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚313787297713664 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚80470181 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 5494538240 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚262 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚313787356763648 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 2755866449408 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚42203387 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 125829120 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 125829120 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚262 _step‚937 C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚4289 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚42203389 JBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚665 % _timestamp‚1.7866616927506845e+09 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 125829120 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 122905600 M@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚59049984 F>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 PIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚57 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1E’>ù2º½O NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚209 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 KDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚60 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚311943408454566 IBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚59 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚429 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 2755824504320 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 4654602240 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚41945088 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚271133552508928 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4289 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚41940992 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 4517085184 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚313787297713664 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚80470181 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 5494538240 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚262 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚313787356763648 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 2755866449408 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚42203387 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 125829120 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 125829120 G?DeviceStatsMonitor.on_train_batch_starاDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 PIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚57 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 @7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚5264 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚223 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚313787356763648 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚35843226 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 2914233396736 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚311031473209344 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 4552982528 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚274047723320832 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚311943466841946 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚322 _runtime‚3041.283872083 G>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4876 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚311031473209344 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚309193322430464 D;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚5264 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 2914191455744 PCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚58387380 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 5368709120 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚322 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚42203387 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 122306526 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚311031490314240 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚271133531865088 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 trainer/global_step‚11399 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚61478912 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚4287 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 2755866449408 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 2755824504320 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚322 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚41348020 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 5494538240 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 PCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚59049984 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚20643840 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 831258624 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 2750086024102 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 F?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚60 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚9568138 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚38266792 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 4552982528 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚9568195 UHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚62584832 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚322 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 4654602240 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17104896 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 122905600 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 845178880 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 5368709120 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚42203389 E8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚80465892 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17104896 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚274047785905664 BDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚313787297713664 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚38262505©Ó[Æ2»- _wandbruntime‚3041  _runtime‚3041‡X‘ È&¼ÂM XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 831258624 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 2750316596714 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚322 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚313813910173184 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 4552982528 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 4517085184 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚322 >5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5264 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚311057812652032 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚311969824820714 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚262 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 845178880 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 NADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚58387428 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 2756055572992 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 2914432620032 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 5494538240 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 5368709120 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚313813910173184 EDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚80477027 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚35846228 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 125829120 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚80477027 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚42206942 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚42206940 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚322 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚4293 NADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚59053056 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17104896 H@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚665 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚20643840 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 DDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59053056 I@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚4876 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 PDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521206 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚309219525263360 _step‚938 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 125829120 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚223 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚61478912 BDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654602240 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 4552982528 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 125829120 _runtime‚3041.443176376 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚9568992 SFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚66776064 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 5494538240 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚262 D=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚60 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 2756097521152 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚311057795547136 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚41948160 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚271156599881728 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚309219508224000 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚38270085 G@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚65 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚35846163 B9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚5264 NGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚63 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚41948160 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚4293 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚4295 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚311057812652032 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚274071011857920 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 5368709120 trainer/global_step‚11399 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 IBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚60 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚42206940 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚38270085 E8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚262 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚209 G@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚60 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚38265792 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚271156579237888ˉ•,Í&½ÇM E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 2756097521152 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚311057795547136 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚41948160 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚271156599881728 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚309219508224000 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚38270085 G@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚65 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚35846163 B9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚5264 NGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚63 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚41948160 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚4293 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚4295 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚311057812652032 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚274071011857920 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 5368709120 trainer/global_step‚11399 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 IBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚60 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚42206940 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚38270085 E8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚262 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚209 G@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚60 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚38265792 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚271156579237888 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 831258624 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 2750316596714 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚322 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚313813910173184 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 4552982528 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 4517085184 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚322 >5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5264 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚311057812652032 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚311969824820714 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚262 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 845178880 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 NADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚58387428 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 2756055572992 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 2914432620032 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 5494538240 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 5368709120 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚313813910173184 EDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚80477027 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚35846228 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 125829120 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚80477027 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚42206942 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚42206940 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚322 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚4293 NADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚59053056 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17104896 H@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚665 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚20643840 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 DDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59053056 I@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚4876 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 PDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521206 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚309219525263360 _step‚938 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 125829120 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚223 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚61478912 BDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654602240 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 4552982528 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 125829120 _runtime‚3041.443176376 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚9568992 SFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚66776064 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 5494538240 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚262 D=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚60ªŸJð2¾- _wandbruntime‚3041  _runtime‚3041r‘£2t¿î epoch‚28 % _timestamp‚1.7866616929140155e+09 _runtime‚ 3041.44385576 0loss/FRt1/assignment_loss‚0.7136967570086171 0loss/FRt2/assignment_loss‚0.6430013848487008 /loss/FRt2/detection_loss‚0.5160657679033314 trainer/global_step‚11399 _step‚939 0loss/FRt1/detection_loss‚0.49590062881575875 &loss/total_loss‚2.3686645385764082çß>yÀó 0loss/FRt1/detection_loss‚0.49590062881575875 &loss/total_loss‚2.3686645385764082 epoch‚28 % _timestamp‚1.7866616929140155e+09 _runtime‚ 3041.44385576 0loss/FRt1/assignment_loss‚0.7136967570086171 0loss/FRt2/assignment_loss‚0.6430013848487008 /loss/FRt2/detection_loss‚0.5160657679033314 trainer/global_step‚11399 _step‚939«_@’Ú¡Á‚(:– À–ùÓïΖá gpu.0.gpu‚61.0 gpu.0.memory‚11, gpu.0.memoryAllocated‚7.0313262939453125) gpu.0.memoryAllocatedBytes‚ 6039863296 gpu.0.temp‚35.0 gpu.0.powerWatts‚132.338' gpu.0.enforcedPowerLimitWatts‚500.0) gpu.0.powerPercent‚26.467599999999997 gpu.0.smClock‚1410 gpu.0.memoryClock‚1593$ gpu.0.correctedMemoryErrors‚1955# gpu.0.uncorrectedMemoryErrors‚0V8‚(:‚ À–ùÓ‚ö”ä disk.sda3.in‚ 28133.140625( disk.nvme0c0n1.out‚57498.17333984375 disk.sda.in‚ 28133.140625 disk.sda2.out‚0 disk.drbd2165.in‚0 disk.drbd2165.out‚0* disk./.usagePercent‚32.767528180490146 disk.sda2.in‚0& disk.nvme0n1.out‚57498.17333984375 disk.dm-1.in‚ 28133.140625 disk.dm-0.in‚ 43647.453125# disk.dm-0.out‚57498.17333984375$ memory_percent‚67.99077987670898 proc.memory.rssMB‚2322.25% disk./.usageGB‚488.25984954833984 disk.sda.out‚ 56.63671875 disk.sda1.in‚0 disk.sda1.out‚0 disk.dm-1.out‚ 56.63671875) proc.memory.percent‚14.17388916015625 disk.sda3.out‚ 56.63671875" disk.nvme0c0n1.in‚ 43647.453125 disk.nvme0n1.in‚ 43647.453125' proc.memory.availableMB‚ 5244.390625 cpu‚27.18563218426271 proc.cpu.threads‚17 network.sent‚ 1.48137e+07 network.recv‚ 5.666448e+06“`¸2Ã- _wandbruntime‚3053  _runtime‚3053œ¥*šÇÄÁ & lr-AdamW/pg1‚0.0006008536490591555 & lr-AdamW/pg2‚0.0006008536490591555 trainer/global_step‚11449 $ _timestamp‚1.786661704942041e+09 _runtime‚3053.472100317 _step‚940't·ïÌÅÆ & lr-AdamW/pg2‚0.0006008536490591555 trainer/global_step‚11449 $ _timestamp‚1.786661704942041e+09 _runtime‚3053.472100317 _step‚940 & lr-AdamW/pg1‚0.0006008536490591555¬›Ëg2Æ- _wandbruntime‚3053  _runtime‚3053vœfǸO SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 4552982528 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚20643840 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚272285900505088 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 2767453215232 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 4552982528 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 2767453215232 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚61478912 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 5494538240 C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚4289 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚429 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 845178880 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚56621056 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 2926454040576 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚313264259941974 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 5368709120 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚312347590098944 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚4287 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚262 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 2761689182730 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 5494538240 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0wßá¼ L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚42381037 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 4654602240 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 122905600 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 BDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 IDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚315115001369088 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4289 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 2926510661632 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 F?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚60 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 122306526 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚322 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 _step‚941 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚9608532 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚313264318329354 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚315115060419072 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 122905600 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚35993813 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 125829120 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17104896 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 PCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚58387380 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚35993734 PCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚59049984 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17104896 KBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚4876 M@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚59049984 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚315115060419072 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 2767411270144 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚322 E8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚80808644 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 2767411270144 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 4654602240 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 125829120 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚322 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚262 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚315115001369088 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 5368709120 IBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚60 % _timestamp‚1.7866617049429934e+09 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 125829120 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚312347590098944 UHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚77264896 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 4628521206 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚310502629146624 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚38431894 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚312347607203840 D;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚5264 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚41348020 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚275212431810560 trainer/global_step‚11449 G>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4876 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚80812933 JBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚665 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 831258624 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 IBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚79 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚42381037 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚310502612107264 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚26385281 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚322 IDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚4287iÜÃ'ȽO M@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚59049984 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚315115060419072 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 2767411270144 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚322 E8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚80808644 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 2767411270144 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 4654602240 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 125829120 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚322 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚262 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚315115001369088 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 5368709120 IBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚60 % _timestamp‚1.7866617049429934e+09 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 125829120 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚312347590098944 UHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚77264896 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 4628521206 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚310502629146624 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚38431894 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚312347607203840 D;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚5264 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚41348020 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚275212431810560 trainer/global_step‚11449 G>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4876 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚80812933 JBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚665 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 831258624 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 IBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚79 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚42381037 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚310502612107264 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚26385281 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚322 IDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚4287 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 4552982528 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚20643840 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚272285900505088 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 2767453215232 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 4552982528 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 2767453215232 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚61478912 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 5494538240 C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚4289 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚429 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 845178880 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚56621056 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 2926454040576 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚313264259941974 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 5368709120 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚312347590098944 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚4287 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚262 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 2761689182730 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 5494538240 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚42381037 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 4654602240 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 122905600 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 BDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 IDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚315115001369088 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4289 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 2926510661632 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 F?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚60 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 122306526 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚322 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 _step‚941 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚9608532 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚313264318329354 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚315115060419072 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 122905600 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚35993813 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 125829120 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17104896 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 PCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚58387380 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚35993734 PCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚59049984 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17104896 KBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚4876­ê7FÜ2É- _wandbruntime‚3053  _runtime‚3053Î8hÅ&Ê¿M REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚41948160 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 122905600 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚35996816 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚42384590 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚35996738 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚275235647145984 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚41348068 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 IBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚60 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 122905600 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 831258624 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚322 trainer/global_step‚11449 # _timestamp‚1.78666170510442e+09 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚42384590 NADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚59053056 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚315141554775552 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚26387484 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 125829120 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚322 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚429 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 4552982528 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚61478912 NADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚58387428 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 5494538240 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 5368709120 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚312373929541632 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚26387486 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚209 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 2761919755390 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚322 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚42384592 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚42384592 I@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚4876 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 5494538240 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17104896 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚272308968521728 _step‚942 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 2767684286976 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚272308947877888 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚313290676308122 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 2761878407322 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚312373912436736 G@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚78 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 2926749594624 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚223 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚313290734695550 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚310528814940160 >5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5264 DDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654602240 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 122306526 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 125829120 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚262 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 845178880 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚38430894 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚312373912436736 B9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚5264 BDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 5368709120 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚80819779 D=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚60 _runtime‚ 3053.63469338 K>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59053056 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚4293 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚9609330 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚38435187 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚38435187 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚322 G@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚60 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 5494538240 EDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521206 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 2767642338816 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚80819779 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚262 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚4293 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚312373929541632 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 4654602240 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 5368709120g7©÷gËÄM D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚38435187 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚38435187 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚322 G@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚60 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 5494538240 EDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521206 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 2767642338816 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚80819779 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚262 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚4293 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚312373929541632 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 4654602240 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 5368709120 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚41948160 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 122905600 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚35996816 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚42384590 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚35996738 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚275235647145984 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚41348068 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 IBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚60 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 122905600 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 831258624 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚322 trainer/global_step‚11449 # _timestamp‚1.78666170510442e+09 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚42384590 NADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚59053056 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚315141554775552 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚26387484 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 125829120 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚322 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚429 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 4552982528 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚61478912 NADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚58387428 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 5494538240 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 5368709120 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚312373929541632 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚26387486 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚209 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 2761919755390 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚322 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚42384592 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚42384592 I@DeviceStatsMonitor.o>êÇcn_train_batch_end/allocation.small_pool.peak‚4876 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 5494538240 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17104896 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚272308968521728 _step‚942 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 2767684286976 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚272308947877888 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚313290676308122 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 2761878407322 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚312373912436736 G@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚78 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 2926749594624 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚223 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚313290734695550 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚310528814940160 >5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5264 DDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654602240 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 122306526 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 125829120 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚262 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 845178880 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚38430894 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚312373912436736 B9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚5264 BDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 5368709120 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚80819779 D=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚60 _runtime‚ 3053.63469338 K>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59053056 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚4293 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚9609330®v®í2Ì- _wandbruntime‚3053  _runtime‚3053˜W2¨rÍì 0loss/FRt2/assignment_loss‚0.5372234575066505 /loss/FRt1/detection_loss‚0.5015816030563668 /loss/FRt2/detection_loss‚0.4994269125933595 % _timestamp‚1.7866617051058266e+09 _runtime‚ 3053.63565821 _step‚943 0loss/FRt1/assignment_loss‚0.5798970553792551 %loss/total_loss‚2.118129028535632 epoch‚28 trainer/global_step‚11449ÿ›wÎñ 0loss/FRt1/assignment_loss‚0.5798970553792551 %loss/total_loss‚2.118129028535632 epoch‚28 trainer/global_step‚11449 0loss/FRt2/assignment_loss‚0.5372234575066505 /loss/FRt1/detection_loss‚0.5015816030563668 /loss/FRt2/detection_loss‚0.4994269125933595 % _timestamp‚1.7866617051058266e+09 _runtime‚ 3053.63565821 _step‚943¯oSá Ï‚(:• Ï–ùÓøè…á gpu.0.gpu‚37.0 gpu.0.memory‚8, gpu.0.memoryAllocated‚7.0313262939453125) gpu.0.memoryAllocatedBytes‚ 6039863296 gpu.0.temp‚33.0 gpu.0.powerWatts‚119.843' gpu.0.enforcedPowerLimitWatts‚500.0) gpu.0.powerPercent‚23.968600000000002 gpu.0.smClock‚1410 gpu.0.memoryClock‚1593$ gpu.0.correctedMemoryErrors‚1955# gpu.0.uncorrectedMemoryErrors‚0F#&:ŠÐ‚(:ÿ Ï–ùÓÁîåã disk.drbd2165.in‚0 disk.drbd2165.out‚0 disk.dm-0.in‚ 43647.453125 disk.sda2.in‚0 disk.nvme0n1.in‚ 43647.453125) proc.memory.availableMB‚ 5244.91796875) proc.memory.percent‚14.17388916015625) disk./.usagePercent‚32.76755608531138 disk.sda3.in‚ 28133.140625$ memory_percent‚67.98756122589111 cpu‚27.185224303612976 proc.cpu.threads‚17 network.sent‚ 1.4876984e+07 disk.dm-1.in‚ 28133.140625" disk.dm-0.out‚58172.4970703125 disk.sda3.out‚ 56.9453125" disk.nvme0c0n1.in‚ 43647.453125 disk.sda.in‚ 28133.140625 disk.sda.out‚ 56.9453125 disk.sda1.in‚0 network.recv‚ 5.686354e+06$ disk./.usageGB‚488.2602653503418 disk.dm-1.out‚ 56.9453125' disk.nvme0c0n1.out‚58172.4970703125 disk.sda1.out‚0 disk.sda2.out‚0% disk.nvme0n1.out‚58172.4970703125 proc.memory.rssMB‚2322.25IomØ2Ñ- _wandbruntime‚3065  _runtime‚3065ŸnÄÄÈÒ % _timestamp‚1.7866617174331908e+09 _runtime‚3065.963206857 _step‚944 & lr-AdamW/pg1‚0.0005949678500375745 & lr-AdamW/pg2‚0.0005949678500375745 trainer/global_step‚11499˜$:‚ÍÓÇ trainer/global_step‚11499 % _timestamp‚1.7866617174331908e+09 _runtime‚3065.963206857 _step‚944 & lr-AdamW/pg1‚0.0005949678500375745 & lr-AdamW/pg2‚0.0005949678500375745°ÕöÆ2Ô- _wandbruntime‚3065  _runtime‚3065HnºÎ¾'Õ¸O ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚276377060983296 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 2778998035968 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚38596996 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 2773250993338 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 4552982528 UHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚68876288 _runtime‚3065.967894677 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚42558687 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 5494538240 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 5368709120 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚36144493 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚38592709 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚313663706988544 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 4552982528 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 2779039981056 KDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚60 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚4287 IDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4876 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 122905600 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 122905600 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 845178880 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚311811918823424 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚322 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚38592709 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 125829120 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚322 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 5368709120 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚20643840 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚273438269145088 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 831258624 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚61478912 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 5494538240 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚262 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚322 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚314585169816762 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 4628521206 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚42558689 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚316442705024512 JBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚665 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 IBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚79 KBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚4876 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚41945088 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚322 E8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚81151396 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚316442764074496 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 2779039981056 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 125829120 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚262 IBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚60 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚9648885 M@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚59049984 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 4654602240 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚81155685 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚314585111429382 IDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚316442705024512 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚26495531 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚38596996 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚313663724093440 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚313663724093440 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4289 BDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 5494538240 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚429 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚316442764074496 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 2773209645318 F?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚60 @7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚5264 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚313663706988544 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚42558687 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 125829120 trainer/global_step‚11499 % _timestamp‚1.7866617174340894e+09 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚42558689 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 2938722961920 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 5368709120[ ¤Ã'Ö½O XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 5368709120 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚36144493 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚38592709 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚313663706988544 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 4552982528 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 2779039981056 KDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚60 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚4287 IDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4876 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 122905600 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 122905600 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 845178880 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚311811918823424 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚322 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚38592709 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 125829120 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚322 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 5368709120 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚20643840 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚273438269145088 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 831258624 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚61478912 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 5494538240 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚262 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚322 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚314585169816762 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 4628521206 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚42558689 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚316442705024512 JBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚665 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 IBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚79 KBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚4876 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚41945088 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚322 E8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚81151396 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚316442764074496 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 2779039981056 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 125829120 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚262 IBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚60 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚9648885 M@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚59049984 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 4654602240 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚81155685 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚314585111429382 IDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚316442705024512 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚26495531 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚38596996 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚313663724093440 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚313663724093440 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4289 BDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 5494538240 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚429 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚316442764074496 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 2773209645318 F?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚60 @7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚5264 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚313663706988544 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚42558687 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 125829120 trainer/global_step‚11499 % _timestamp‚1.7866617174340894e+09 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚42558689 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 2938722961920 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 5368709120 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚276377060983296 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 2778998035968 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚38596996 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 2773250993338 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 4552982528 UHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚68876288 _runtime‚3065.967894677 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚42558687 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 5494538240 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0±Ñÿ{2×- _wandbruntime‚3066  _runtime‚3066UóJJØÂM H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚4293 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚38595996 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚36147421 NGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚67 SFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚66776064 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚313690046431232 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 4552982528 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚41948160 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚81158236 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚9649687 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚209 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 845178880 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚316469258430976 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 122905600 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚276400353187328 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521206 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚322 >5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5264 DDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 122905600 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚42562240 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚276400286411264 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚262 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 2779271052800 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚4293 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 2773440217930 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚38595996 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 2779271052800 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚46132224 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17104896 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚316469317484032 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚313690046431232 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚322 _step‚946 B9DevÞf³~iceStatsMonitor.on_train_batch_end/allocation.all.peak‚5264 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚322 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚314611586182958 K>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59053056 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚313690029326336 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 831258624 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚262 PDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 trainer/global_step‚11499 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚314611527795530 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 5368709120 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 125829120 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚313690029326336 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚273461316517888 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚81162531 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚9649754 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 5368709120 G@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚60 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654602240 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 5494538240 IBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚60 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚42562242 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚26497736 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚42562242 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚316469317484032 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 4654602240 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚36147490 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 _runtime‚3066.127425345 E6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚322 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚314611586182958 K>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59053056 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚313690029326336 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 831258624 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚262 PDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 trainer/global_step‚11499 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚314611527795530 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 5368709120 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 125829120 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚313690029326336 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚273461316517888 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚81162531 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚9649754 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 5368709120 G@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚60 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654602240 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 5494538240 IBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚60 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚42562242 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚26497736 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚42562242 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚316469317484032 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 4654602240 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚36147490 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 _runtime‚3066.127425345 EDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚36147421 NGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚67 SFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚66776064 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚313690046431232 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 4552982528 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚41948160 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚81158236 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚9649687 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚209 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 845178880 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚316469258430976 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 122905600 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚276400353187328 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521206 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚322 >5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5264 DDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 122905600 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚42562240 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚276400286411264 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚262 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 2779271052800 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚4293 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 2773440217930 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚38595996 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 2779271052800 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚46132224²Si³2Ú- _wandbruntime‚3066  _runtime‚3066­@iôsÛí %loss/total_loss‚2.160451554753364 0loss/FRt1/assignment_loss‚0.6270104401428334 epoch‚28 trainer/global_step‚11499 $ _timestamp‚1.786661717598087e+09 _runtime‚3066.127892337 _step‚947 0loss/FRt2/assignment_loss‚0.5349264431074248 /loss/FRt1/detection_loss‚0.5096890782358761 0loss/FRt2/detection_loss‚0.48882559326722974&yUxÜò _step‚947 0loss/FRt2/assignment_loss‚0.5349264431074248 /loss/FRt1/detection_loss‚0.5096890782358761 0loss/FRt2/detection_loss‚0.48882559326722974 %loss/total_loss‚2.160451554753364 0loss/FRt1/assignment_loss‚0.6270104401428334 epoch‚28 trainer/global_step‚11499 $ _timestamp‚1.786661717598087e+09 _runtime‚3066.127892337³AšÒà•Ý‚(:Š Þ–ùӹšá gpu.0.gpu‚64.0 gpu.0.memory‚12, gpu.0.memoryAllocated‚7.0313262939453125) gpu.0.memoryAllocatedBytes‚ 6039863296 gpu.0.temp‚36.0 gpu.0.powerWatts‚215.535' gpu.0.enforcedPowerLimitWatts‚500.0 gpu.0.powerPercent‚43.107 gpu.0.smClock‚1410 gpu.0.memoryClock‚1593$ gpu.0.correctedMemoryErrors‚1955# gpu.0.uncorrectedMemoryErrors‚0~— „Þ‚(:ù Þ–ùÓ¦øã network.sent‚ 1.4944944e+07 disk.sda.out‚ 57.2578125 disk.sda2.out‚0 disk.nvme0n1.in‚ 43647.453125# disk.nvme0n1.out‚58179.63671875$ memory_percent‚67.98603534698486 proc.memory.rssMB‚2322.25 cpu‚27.185052057116774 disk.drbd2165.out‚0 disk.dm-0.in‚ 43647.453125% disk.nvme0c0n1.out‚58179.63671875 disk.dm-1.in‚ 28133.140625) proc.memory.availableMB‚ 5245.16796875) proc.memory.percent‚14.17388916015625 proc.cpu.threads‚17) disk./.usagePercent‚32.76755838937919 disk.dm-0.out‚58179.63671875 disk.sda3.in‚ 28133.140625" disk.nvme0c0n1.in‚ 43647.453125 disk.dm-1.out‚ 57.2578125 network.recv‚ 5.711023e+06$ disk./.usageGB‚488.2602996826172 disk.drbd2165.in‚0 disk.sda3.out‚ 57.2578125 disk.sda.in‚ 28133.140625 disk.sda1.in‚0 disk.sda1.out‚0 disk.sda2.in‚0p$™¢2ß- _wandbruntime‚3078  _runtime‚3078¹Î¹’ÇàÁ trainer/global_step‚11549 % _timestamp‚1.7866617296519933e+09 _runtime‚ 3078.18209834 _step‚948 & lr-AdamW/pg1‚0.0005890920084412845 & lr-AdamW/pg2‚0.0005890920084412845e“´ÌáÆ _runtime‚ 3078.18209834 _step‚948 & lr-AdamW/pg1‚0.0005890920084412845 & lr-AdamW/pg2‚0.0005890920084412845 trainer/global_step‚11549 % _timestamp‚1.7866617296519933e+09´4²d®2â- _wandbruntime‚3078  _runtime‚3078óÀük½'ã·O SGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚9689699 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 831258624 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 2951069835776 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚322 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 5368709120 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚262 PCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚59049984 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 122905600 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚223 F?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚60 $ _timestamp‚1.786661729653033e+09 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚314979823878144 D;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚5264 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚274590637785088 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚38762098 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚314979840983040 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚38762098 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚26605779 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚9689756 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 4654602240 _step‚949 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 845178880 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚315905962916790 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚262 G>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4876 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17104896 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 5494538240 M@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚59049984 IBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚59 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 4628521206 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 125829120 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚209 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚39843840 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚262 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 125829120 JBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚665 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚4287 KBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚4876 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚317770467729920 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 5494538240 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 KDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚60 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 4552982528 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚81498437 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 5368709120 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚314979840983040 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚36295478 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 4654602240 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚42736337 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 2784812803946 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 IDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚314979823878144 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 5368709120 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚429 BDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 2790584801792 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚277541728264704 UHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚60487680 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚322 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚317770408679936 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚42736339 PIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚57 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 4517085184 IBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚60 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17104896 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 2790626746880 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 125829120 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚322 @7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚5264 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 2790584801792 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚42736337 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚277541667777024 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚38757811 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚41945088 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚41945088 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4289 IDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4876 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17104896 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 5494538240 M@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚59049984 IBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚59 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 4628521206 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 125829120 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚209 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚39843840 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚262 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 125829120 JBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚665 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚4287 KBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚4876 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚317770467729920 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 5494538240 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 KDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚60 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 4552982528 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚81498437 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 XIDeviceSºÇ¬HRtatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 5368709120 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚314979840983040 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚36295478 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 4654602240 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚42736337 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 2784812803946 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 IDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚314979823878144 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 5368709120 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚429 BDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 2790584801792 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚277541728264704 UHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚60487680 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚322 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚317770408679936 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚42736339 PIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚57 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 4517085184 IBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚60 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17104896 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 2790626746880 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 125829120 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚322 @7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚5264 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 2790584801792 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚42736337 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚277541667777024 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚38757811 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚41945088 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚41945088 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4289 IDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521206 K>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59053056 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 2790857818624 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚81500988 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚313147377254400 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚41348068 % _timestamp‚1.7866617298141737e+09 NADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚59053056 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚81505283 SFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚66776064 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚322 H@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚665 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17104896 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚315006146215936 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚322 _runtime‚3078.344487845 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚42739892 IBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚60 P8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚315006163320832 I@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚4876 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚36298581 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚274613705801728 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚315932379282938 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 2785043376606 D=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚60 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚4293 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 4552982528 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 4552982528 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 5368709120 G@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚60 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚81505283 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚41948160 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚277565022926336 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 5368709120 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚38761098 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 >5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5264 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚9690522 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 122306526 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚277564956150272 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚46132224 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚429 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚4295 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚42739890 DDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654602240 B9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚5264 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚36298506 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚315932437670366 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 5494538240 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚262 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚317796962086400 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚38765391 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 2790815870464 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚61478912 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚38761098 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚42739890 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚223 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚209 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚20643840 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚262 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 122905600 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 5494538240 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 5494538240 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚42739892 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚313147394293760 _step‚950 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 4517085184 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 5368709120 EDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17104896 G@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚75 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚322 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 125829120 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚322 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚4293 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚26607984 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚317797021139456 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 122905600 EDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚42739892 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚313147394293760 _step‚950 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 4517085184 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 5368709120 EDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17104896 G@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚75 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚322 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 125829120 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚322 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚4293 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚26607984 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚317797021139456 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 122905600 EDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521206 K>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59053056 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 2790857818624 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚81500988 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚313147377254400 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚41348068 % _timestamp‚1.7866617298141737e+09 NADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚59053056 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚81505283 SFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚66776064 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚322 H@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚665 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17104896 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚315006146215936 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚322 _runtime‚3078.344487845 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚42739892 IBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚60 P8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚315006163320832 I@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚4876 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚36298581 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚274613705801728 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚315932379282938 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 2785043376606 D=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚60 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚4293 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 4552982528 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 4552982528 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 5368709120 G@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚60 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚81505283 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚41948160 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚277565022926336 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 5368709120 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚38761098 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 >5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5264 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚9690522 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 122306526 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚277564956150272 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚46132224 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚429 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚4295 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚42739890 DDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654602240 B9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚5264 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚36298506 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚315932437670366 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 5494538240 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚262 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚317796962086400 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚38765391 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 2790815870464 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚61478912 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚38761098 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚42739890 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚223 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚209 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚20643840 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚262 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 122905600 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 5494538240¶M‡BÊ2è- _wandbruntime‚3078  _runtime‚30786séí trainer/global_step‚11549 _runtime‚3078.344955328 _step‚951 0loss/FRt1/assignment_loss‚0.5570360625611852 %loss/total_loss‚2.016102544174073 % _timestamp‚1.7866617298151343e+09 0loss/FRt2/assignment_loss‚0.5028854020818991 /loss/FRt1/detection_loss‚0.4717389584456923 /loss/FRt2/detection_loss‚0.4844421210852965 epoch‚28*Jùÿxêò 0loss/FRt2/assignment_loss‚0.5028854020818991 /loss/FRt1/detection_loss‚0.4717389584456923 /loss/FRt2/detection_loss‚0.4844421210852965 epoch‚28 trainer/global_step‚11549 _runtime‚3078.344955328 _step‚951 0loss/FRt1/assignment_loss‚0.5570360625611852 %loss/total_loss‚2.016102544174073 % _timestamp‚1.7866617298151343e+09·Q$Ñ$¡ë‚(:– í–ùÓÂëšá gpu.0.gpu‚62.0 gpu.0.memory‚12, gpu.0.memoryAllocated‚7.0313262939453125) gpu.0.memoryAllocatedBytes‚ 6039863296 gpu.0.temp‚35.0 gpu.0.powerWatts‚130.989' gpu.0.enforcedPowerLimitWatts‚500.0) gpu.0.powerPercent‚26.197799999999997 gpu.0.smClock‚1410 gpu.0.memoryClock‚1593$ gpu.0.correctedMemoryErrors‚1955# gpu.0.uncorrectedMemoryErrors‚0ØÛ¨Žì‚(:ƒ í–ùÓ†˜ã disk.sda3.in‚ 28133.140625( disk.nvme0c0n1.out‚58184.05419921875 disk.sda1.out‚0 network.sent‚ 1.5014616e+07 disk.nvme0n1.in‚ 43647.453125& disk.nvme0n1.out‚58184.05419921875 disk.sda1.in‚0 disk.sda2.in‚0$ memory_percent‚67.98844337463379( proc.memory.availableMB‚ 5244.7734375 proc.memory.rssMB‚2322.25" disk.nvme0c0n1.in‚ 43647.453125) proc.memory.percent‚14.17388916015625 cpu‚27.184577464317623 proc.cpu.threads‚17) disk./.usagePercent‚32.76735409536686 disk.dm-1.in‚ 28133.140625 disk.dm-1.out‚ 57.48046875# disk.dm-0.out‚58184.05419921875 disk.sda3.out‚ 57.48046875 disk.sda.in‚ 28133.140625 disk.sda.out‚ 57.48046875 disk.sda2.out‚0 network.recv‚ 5.73789e+06$ disk./.usageGB‚488.2572555541992 disk.drbd2165.in‚0 disk.drbd2165.out‚0 disk.dm-0.in‚ 43647.453125oa¥Ë2í- _wandbruntime‚3090  _runtime‚3090fé!ÆîÀ _runtime‚ 3090.5918608 _step‚952 & lr-AdamW/pg1‚0.0005832265016646106 & lr-AdamW/pg2‚0.0005832265016646106 trainer/global_step‚11599 % _timestamp‚1.7866617420618567e+09€CÏhËïÅ _step‚952 & lr-AdamW/pg1‚0.0005832265016646106 & lr-AdamW/pg2‚0.0005832265016646106 trainer/global_step‚11599 % _timestamp‚1.7866617420618567e+09 _runtime‚ 3090.5918608¸IV-2ð- _wandbruntime‚3090  _runtime‚3090JïõÄDñ¸O VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 4552982528 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 2796333266534 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚41945088 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 5368709120 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚316295940767744 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚42913989 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 4654602240 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚61478912 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚262 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚42913989 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚322 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚317226814404198 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 125829120 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚429 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚26716031 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 5494538240 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 125829120 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 125829120 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚322 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚38927200 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚42913987 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚223 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 F?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚60 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚41348020 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 4517085184 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚262 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚319098171385344 PCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚59049984 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚46135296 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 5368709120 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚322 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 trainer/global_step‚11599 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17104896 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 2802171567616 KBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚4876 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚36446749 C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚4289 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚319098112335360 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚316295957872640 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 122905600 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚4287 ZFDeviceStatsMonitor.on_train_batch_start/¸‘zinactive_split_bytes.all.freed‚278706326171136 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 2963365881344 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚42913987 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚316295957872640 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 2802213512704 IDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 G>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4876 PCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚58387380 _step‚953 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚38922913 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17104896 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 4552982528 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 2802171567616 D;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚5264 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 5494538240 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 122905600 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 4628521206 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚314430481137664 UHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚66779136 BDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 122306526 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚38927200 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚209 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚262 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚278706392950272 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚275743027068928 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 4654602240 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚9730652 IDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚319098112335360 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚26716029 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚275743006425088 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 % _timestamp‚1.7866617420627737e+09 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚81841189 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚4287 M@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚59049984 E8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚81836900 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 2963319746048 KDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚60 IBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚60 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4289 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 5368709120ÿxÃ'ò½O YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚316295940767744 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚42913989 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 4654602240 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚61478912 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚262 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚42913989 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚322 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚317226814404198 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 125829120 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚429 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚26716031 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 5494538240 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 125829120 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 125829120 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚322 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚38927200 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚42913987 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚223 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 F?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚60 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚41348020 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 4517085184 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚262 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚319098171385344 PCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚59049984 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚46135296 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 5368709120 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚322 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 trainer/global_step‚11599 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17104896 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 2802171567616 KBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚4876 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚36446749 C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚4289 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚319098112335360 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚316295957872640 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 122905600 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚4287 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚278706326171136 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 2963365881344 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚42913987 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚316295957872640 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 2802213512704 IDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 G>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4876 PCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚58387380 _step‚953 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚38922913 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17104896 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 4552982528 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 2802171567616 D;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚5264 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 5494538240 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 122905600 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 4628521206 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚314430481137664 UHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚66779136 BDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 122306526 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚38927200 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚209 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚262 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚278706392950272 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚275743027068928 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 4654602240 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚9730652 IDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚319098112335360 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚26716029 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚275743006425088 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 % _timestamp‚1.7866617420627737e+09 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚81841189 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚4287 M@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚59049984 E8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚81836900 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 2963319746048 KDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚60 IBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚60 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4289 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 5368709120 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 4552982528 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 2796333266534 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚41945088 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 5368709120¹ä>Ïg2ó- _wandbruntime‚3090  _runtime‚3090ßÉ È&ôÂM ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚316322280210432 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 % _timestamp‚1.7866617422239902e+09 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚41948160 G@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚57 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚278729614183936 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚317253289157774 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚314456666931200 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 125829120 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚4293 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚50326528 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 5368709120 P5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5264 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚81848035 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚9731490 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 125829120 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚38930493 K>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59053056 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚275766053797888 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 122306526 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 125829120 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚4295 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚4293 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚81843740 SFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚70970368 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 2963610712576 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚314456683970560 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654602240 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚209 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 NADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚59053056 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚61478912 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚41348068 D=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚60 B9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚5264 H@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚665 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚278729685154304 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 _runtime‚3090.754251786 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚319124724794880 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚319124665741824 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚429 BDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚36449724 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 831258624 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚26718236 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚42917542 I@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚4876 NGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚55 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 845178880 NADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚58387428 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚262 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚223 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521206 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚322 trainer/global_step‚11599 E8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚317253230770346 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚42917540 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚316322263105536 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚322 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚38926200 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 122905600 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚36449781 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚322 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17104896 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 2963560386048 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚322 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚262 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 4552982528 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 5494538240 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17104896 EDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 G@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚60 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚316322280210432 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 5368709120 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 2796605187214 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚316322263105536 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚42917542 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0Ñ~Òj¦õÇM KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚38930493 K>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59053056 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚275766053797888 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 122306526 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 125829120 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚4295 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚4293 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚81843740 SFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚70970368 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 2963610712576 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚314456683970560 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654602240 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚209 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 NADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚59053056 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚61478912 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚41348068 D=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚60 B9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚5264 H@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚665 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚278729685154304 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 _runtime‚3090.754251786 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚319124724794880 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚319124665741824 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚429 BDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚36449724 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 831258624 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚26718236 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚42917542 I@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚4876 NGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚55 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 845178880 NADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚58387428 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚262 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚223 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521206 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚322 trainer/global_step‚11599 E8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚317253230770346 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚42917540 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚316322263105536 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚322 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚38926200 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 122905600 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚36449781 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚322 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17104896 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 2963560386048 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚322 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚262 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 4552982528 L=DeviceStatsMonitor.on_train_batch_end/reserved_byt­` ' es.all.peak‚ 5494538240 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17104896 EDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 G@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚60 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚316322280210432 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 5368709120 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 2796605187214 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚316322263105536 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚42917542 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚316322280210432 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 % _timestamp‚1.7866617422239902e+09 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚41948160 G@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚57 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚278729614183936 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚317253289157774 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚314456666931200 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 125829120 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚4293 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚50326528 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 5368709120 P5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5264 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚81848035 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚9731490 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 125829120ºx§d¸2ö- _wandbruntime‚3090  _runtime‚3090 ™t÷î 0loss/FRt1/assignment_loss‚0.7008414906081704 0loss/FRt2/assignment_loss‚0.5676873851468435 &loss/total_loss‚2.2503680179536394 epoch‚28 trainer/global_step‚11599 _step‚955 /loss/FRt1/detection_loss‚0.5123130763843003 /loss/FRt2/detection_loss‚0.4695260658143252 % _timestamp‚1.7866617422250462e+09 _runtime‚3090.755185149a½,2yøó 0loss/FRt1/assignment_loss‚0.7008414906081704 0loss/FRt2/assignment_loss‚0.5676873851468435 &loss/total_loss‚2.2503680179536394 epoch‚28 trainer/global_step‚11599 _step‚955 /loss/FRt1/detection_loss‚0.5123130763843003 /loss/FRt2/detection_loss‚0.4695260658143252 % _timestamp‚1.7866617422250462e+09 _runtime‚3090.755185149» / X2ù- _wandbruntime‚3095  _runtime‚3095¢ÏÞÞúØ +particle/sensitivity‚0.6901741027832031 *jet/accuracy_1_of_2‚0.5970063209533691 Purity/0FRt/event_purity‚1 3Purity/1FRt/event_proportion‚0.6297432780265808 'particle/accuracy‚0.632382333278656 trainer/global_step‚11599 )loss/val_total_loss‚2.392435857817522 0Purity/1FRt/event_purity‚0.48337027430534363 0Purity/2FRt/event_purity‚0.44167059659957886 -Purity/2FRt/FRt_purity‚0.5821900367736816 0Purity/*FRt/event_purity‚0.48100993037223816 *jet/accuracy_2_of_2‚0.2256145030260086 /particle/accuracy_2_of_2‚0.7551392316818237 7validation_average_jet_accuracy‚0.32838207483291626 epoch‚28 2Purity/*FRt/event_proportion‚0.669805645942688 particle/accuracy_1_of_1‚1 /particle/accuracy_1_of_2‚0.9401971697807312 *validation_accuracy‚0.2256145030260086 _runtime‚3095.703594955 _step‚956 .Purity/1FRt/FRt_purity‚0.48337027430534363 2Purity/2FRt/event_proportion‚0.040062315762043 'particle/f_score‚0.5713117122650146 .Purity/*FRt/FRt_purity‚0.49467918276786804 3Purity/0FRt/event_proportion‚0.3301943838596344 *jet/accuracy_1_of_1‚0.3230665326118469 +particle/specificity‚0.6005760431289673 $ _timestamp‚1.786661747173546e+09ORÓãûÝ +particle/specificity‚0.6005760431289673 $ _timestamp‚1.786661747173546e+09 +particle/sensitivity‚0.6901741027832031 *jet/accuracy_1_of_2‚0.5970063209533691 Purity/0FRt/event_purity‚1 3Purity/1FRt/event_proportion‚0.6297432780265808 'particle/accuracy‚0.632382333278656 trainer/global_step‚11599 )loss/val_total_loss‚2.392435857817522 0Purity/1FRt/event_purity‚0.48337027430534363 0Purity/2FRt/event_purity‚0.44167059659957886 -Purity/2FRt/FRt_purity‚0.5821900367736816 0Purity/*FRt/event_purity‚0.48100993037223816 *jet/accuracy_2_of_2‚0.2256145030260086 /particle/accuracy_2_of_2‚0.7551392316818237 7validation_average_jet_accuracy‚0.32838207483291626 epoch‚28 2Purity/*FRt/event_proportion‚0.669805645942688 particle/accuracy_1_of_1‚1 /particle/accuracy_1_of_2‚0.9401971697807312 *validation_accuracy‚0.2256145030260086 _runtime‚3095.703594955 _step‚956 .Purity/1FRt/FRt_purity‚0.48337027430534363 2Purity/2FRt/event_proportion‚0.040062315762043 'particle/f_score‚0.5713117122650146 .Purity/*FRt/FRt_purity‚0.49467918276786804 3Purity/0FRt/event_proportion‚0.3301943838596344 *jet/accuracy_1_of_1‚0.3230665326118469¼å\ 6–ü‚(:‹ ü–ùÓ’®žá gpu.0.gpu‚59.0 gpu.0.memory‚11, gpu.0.memoryAllocated‚7.0313262939453125) gpu.0.memoryAllocatedBytes‚ 6039863296 gpu.0.temp‚33.0 gpu.0.powerWatts‚119.703' gpu.0.enforcedPowerLimitWatts‚500.0 gpu.0.powerPercent‚23.9406 gpu.0.smClock‚1410 gpu.0.memoryClock‚1593$ gpu.0.correctedMemoryErrors‚1955# gpu.0.uncorrectedMemoryErrors‚0×ni•ý‚(:Š ü–ùÓÃ÷‡ã disk.sda2.in‚0 disk.sda.in‚28197.54296875 disk.sda.out‚ 57.7421875 disk.dm-1.in‚28197.54296875 disk.drbd2165.in‚0 disk.dm-0.in‚ 43662.3671875) proc.memory.availableMB‚ 5306.19921875# proc.memory.rssMB‚ 2260.64453125 disk.sda3.in‚28197.54296875% disk.nvme0c0n1.out‚58219.49609375 disk.sda2.out‚0 disk.dm-1.out‚ 57.7421875 disk.drbd2165.out‚0$ memory_percent‚67.61353015899658* proc.memory.percent‚13.797879219055176 proc.cpu.threads‚17 network.sent‚ 1.5084179e+07 network.recv‚ 5.764567e+06$ disk./.usageGB‚488.2612838745117 disk.sda3.out‚ 57.7421875 disk.sda1.in‚0 disk.sda1.out‚0! disk.nvme0n1.in‚ 43662.3671875# disk.nvme0n1.out‚58219.49609375* disk./.usagePercent‚32.767624439323015# disk.nvme0c0n1.in‚ 43662.3671875 disk.dm-0.out‚58219.49609375 cpu‚27.191336948795193°«ƒ÷2þ- _wandbruntime‚3108  _runtime‚3108í ˆÀÈÿ _runtime‚3108.510078129 _step‚957 & lr-AdamW/pg1‚0.0005773717064380935 & lr-AdamW/pg2‚0.0005773717064380935 trainer/global_step‚11649 % _timestamp‚1.7866617599800594e+09EŒñ%Í€Ç & lr-AdamW/pg2‚0.0005773717064380935 trainer/global_step‚11649 % _timestamp‚1.7866617599800594e+09 _runtime‚3108.510078129 _step‚957 & lr-AdamW/pg1‚0.0005773717064380935½°˜Æ2- _wandbruntime‚3108  _runtime‚3108K)@3½'‚·O PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚43152245 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚43152243 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚316120687345664 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 4517085184 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚26866895 G>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4876 _runtime‚3108.514869757 F>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚320820879982080 IDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 5494538240 _step‚958 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚39110207 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚316120670306304 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚320820820932096 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 2817826059776 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚39114494 $ _timestamp‚1.786661759980961e+09 PIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚68 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 845178880 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4289 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 5368709120 KDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚60 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 2979764031488 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 2812002397268 D;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚5264 KBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚4876 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚318932689742932 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚39110207 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚209 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚429 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚39114494 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17104896 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 4654602240 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 2979711604736 IDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4876 _runtime‚3108.514869757 F>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚320820879982080 IDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 5494538240 _step‚958 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚39110207 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚316120670306304 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚320820820932096 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 2817826059776 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚39114494 $ _timestamp‚1.786661759980961e+09 PIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚68 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 845178880 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4289 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 5368709120 KDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚60 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 2979764031488 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 2812002397268 D;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚5264 KBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚4876 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚318932689742932 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚39110207 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚209 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚429 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚39114494 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17104896 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 4654602240 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 2979711604736 IDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚36650812 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚318959047721700 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚82273585 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚26869100 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚26869098 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚82269290 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚318029317210112 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚36650884 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚223 NADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚58387428 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚280197647448064 G@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚72 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚4295 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17104896 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚39113494 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚318029317210112 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚318959106109128 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚322 trainer/global_step‚11649 _step‚959 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚429 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚61478912 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚43155796 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚39113494 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚209 H@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚665 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 4654602240 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚41948160 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚280197724709888 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚82273585 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 4552982528 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚43155798 I@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚4876 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚9781784 NGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚70 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 831258624 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 2979954702336 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 5368709120 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 UHDeviceStatsMonitor.on_train_batch[-ŒlM_end/allocated_bytes.large_pool.current‚17104896 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚320847374338560 EDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59053056 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚318029334315008 NADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚59053056 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚322 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 >5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5264 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 122905600 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654602240 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 845178880 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚41348068 DDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521206 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 122306526 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚262 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 5494538240 P6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚322 G@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚60 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚20643840 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 125829120 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚41948160 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 2818099076608 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 2812191621860 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚322 IBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚60 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 5368709120æ,F×Í&†ÇM _runtime‚3108.671841214 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚4293 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚318029334315008 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521206 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 122306526 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚262 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 5494538240 P6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚322 G@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚60 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚20643840 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 125829120 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚41948160 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 2818099076608 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 2812191621860 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚322 IBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚60 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 5368709120 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 2818057128448 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 122905600 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚36650812 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚318959047721700 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚82273585 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚26869100 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚26869098 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚82269290 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚318029317210112 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚36650884 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚223 NADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚58387428 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚280197647448064 G@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚72 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚4295 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17104896 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚39113494 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚318029317210112 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚318959106109128 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚322 trainer/global_step‚11649 _step‚959 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚429 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚61478912 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚43155796 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚39113494 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚209 H@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚665 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 4654602240 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚41948160 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚280197724709888 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚82273585 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 4552982528 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚43155798 I@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚4876 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚9781784 NGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚70 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 831258624 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 2979954702336 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 5368709120 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17104896 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚320847374338560 EDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59053056 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚318029334315008 NADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚59053056 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚322 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 >5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5264 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 122905600 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654602240 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 845178880 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚41348068 DDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚43329895 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚322148524587520 UHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚75167744 PCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚58387380 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚41348020 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚262 _step‚962 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚9821942 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚20643840 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚278347014012928 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚43329895 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚429 _runtime‚3120.666988051 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚41945088 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 4552982528 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚39275309 PIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚72 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 845178880 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 5494538240 E8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚82605202 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚39279596 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚223 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 2823522859856 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 125829120 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 4654602240 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 4628521206 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚317429959983104 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚43329893 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17104896 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚41945088 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚4287 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 122306526 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 5368709120 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 KDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚60 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 125829120 C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚4289 IDeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4289 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚278347034656768 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 2829454770688 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 @7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚5264 D;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚5264 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 2992029397504 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚322 $ _timestamp‚1.786661772135743e+09 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 4552982528 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚43329893 G>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4876 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚54523904 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 2991974873600 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚61478912 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 IBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚60 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚281338988886528 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚322 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 122905600 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚322148524587520 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚319319128866816 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚322 IDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚317429977022464 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 4517085184 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 2823564207876 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 831258624 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚262 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 2829412825600 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 122905600 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 5368709120 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚39275309 M@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚59049984 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 2829412825600 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0îûÅ—‘¼O WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 2829412825600 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 122905600 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 5368709120 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚39275309 M@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚59049984 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 2829412825600 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚320253482842960 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 5368709120 IBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚74 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 4654602240 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 2829454770688 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚319319128866816 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚319319111761920 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚322148583637504 KBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚4876 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚209 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚9821870 trainer/global_step‚11699 F>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚43329895 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚322148524587520 UHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚75167744 PCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚58387380 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚41348020 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚262 _step‚962 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚9821942 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚20643840 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚278347014012928 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚43329895 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚429 _runtime‚3120.666988051 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚41945088 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 4552982528 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚39275309 PIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚72 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 845178880 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 5494538240 E8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚82605202 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚39279596 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚223 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 2823522859856 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.3Y›+allocated‚ 125829120 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 4654602240 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 4628521206 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚317429959983104 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚43329893 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17104896 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚41945088 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚4287 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 122306526 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 5368709120 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 KDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚60 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 125829120 C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚4289 IDeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4289 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚278347034656768 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 2829454770688 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 @7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚5264 D;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚5264 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 2992029397504 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚322 $ _timestamp‚1.786661772135743e+09 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 4552982528 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚43329893 G>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4876 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚54523904 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 2991974873600 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚61478912 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 IBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚60 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚281338988886528 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚322 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 122905600 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚322148524587520 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚319319128866816 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚322 IDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚317429977022464 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 4517085184 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 2823564207876 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 831258624 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚262ÂÆ2’- _wandbruntime‚3120  _runtime‚3120@”.«Ç&“ÁM KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚322 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17104896 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 2829685842432 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚278370082029568 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚320279957596536 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚4295 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚39278596 E6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚322 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 _runtime‚3120.826560858 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 4552982528 NADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚58387428 G@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚60 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 831258624 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚262 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚41948160 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚262 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚82612042 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚43333448 G@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚74 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚322175137047040 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚281362282699264 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 BDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59053056 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚4293 EDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 5494538240 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 2829685842432 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 H@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚665 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚429 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 4517085184 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚322175077993984 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚319345451204608 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚36802052 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 5494538240 DDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521206 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚4293 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚26979350 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 5494538240 IBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚60 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚223 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚317456162816000 P5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5264 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654602240 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚319345451204608 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚43333448 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 125829120 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚41948160 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚39278596 SFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚64678912 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 trainer/global_step‚11699 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 2992221313536 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 5368709120 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 125829120 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚39282889 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 2823753432468 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 125829120 _step‚963 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚319345434099712 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 4552982528 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚82616337 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚82612042 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 2823794780536 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 D=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚60 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 5368709120 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚278370061385728 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 2992265348608 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚322 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚39282889 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚44035072 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 5368709120 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚262 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚82616337 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 2829643894272 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚319345434099712 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚281362347378176 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚317456145776640 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 122306526Ÿ!ÕÌ&”ÆM DDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521206 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚4293 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚26979350 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 5494538240 IBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚60 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚223 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚317456162816000 P5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5264 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654602240 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚319345451204608 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚43333448 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 125829120 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚41948160 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚39278596 SFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚64678912 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 trainer/global_step‚11699 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 2992221313536 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 5368709120 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 125829120 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚39282889 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 2823753432468 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 125829120 _step‚963 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚319345434099712 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 4552982528 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚82616337 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚82612042 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 2823794780536 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 D=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚60 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 5368709120 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚278370061385728 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 2992265348608 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚322 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚39282889 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚44035072 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 5368709120 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚262 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚82616337 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 2829643894272 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚319345434099712 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚281362347378176 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚317456145776640 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 122306526 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚322 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17104896 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 2829685842432 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚278370082029568 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚320279957596536 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚4295 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚39278596 E6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚322 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 _runtime‚3120.826560858 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 4552982528 NADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚58387428 G@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚60 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 831258624 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚262 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚41948160 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚262 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚82612042 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚43333448 G@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚74 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚322175137047040 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚281362282699264 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 BDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59053056 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚4293 EDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 5494538240 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 2829685842432 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 H@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚665 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚429 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 4517085184 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚322175077993984 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚319345451204608 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚36802052 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 5494538240Ã=Ïx 2•- _wandbruntime‚3120  _runtime‚3120u/tPu–ï /loss/FRt1/detection_loss‚0.5273397022197636 0loss/FRt2/detection_loss‚0.46517386167411195 epoch‚29 % _timestamp‚1.7866617722972422e+09 _runtime‚3120.827052225 _step‚964 0loss/FRt1/assignment_loss‚0.6042251586870676 %loss/total_loss‚2.085995695096985 trainer/global_step‚11699 1loss/FRt2/assignment_loss‚0.48925697251604156œQ¾z—ô % _timestamp‚1.7866617722972422e+09 _runtime‚3120.827052225 _step‚964 0loss/FRt1/assignment_loss‚0.6042251586870676 %loss/total_loss‚2.085995695096985 trainer/global_step‚11699 1loss/FRt2/assignment_loss‚0.48925697251604156 /loss/FRt1/detection_loss‚0.5273397022197636 0loss/FRt2/detection_loss‚0.46517386167411195 epoch‚29ÄÀž5Ð2˜- _wandbruntime‚3132  _runtime‚3132ŽoüçÆ™À trainer/global_step‚11749 # _timestamp‚1.78666178428513e+09 _runtime‚3132.815153487 _step‚965 & lr-AdamW/pg1‚0.0005656957540936205 & lr-AdamW/pg2‚0.0005656957540936205íœøq˚Š# _timestamp‚1.78666178428513e+09 _runtime‚3132.815153487 _step‚965 & lr-AdamW/pg1‚0.0005656957540936205 & lr-AdamW/pg2‚0.0005656957540936205 trainer/global_step‚11749Ŵ鬚2›- _wandbruntime‚3132  _runtime‚3132Z)>¦œ¸O ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 2841041536512 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚82952243 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚52426752 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 F?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚60 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 4552982528 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 IBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚68 G>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4876 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚41945088 PIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚66 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚61478912 _step‚966 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚4287 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚27087395 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚223 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 2840999591424 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚9862953 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚282503623514112 KDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚60 IBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚60 _runtime‚3132.819926834 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚323476287292928 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚320635245756416 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚323476228242944 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚321574334330368 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 4517085184 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 2841041536512 KBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚4876 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚262 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚429 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚27087393 PCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚58387380 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚9862887 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚322 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚39440411 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 2840999591424 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚282503696584704 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚279499403296768 BDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚323476228242944 QDDeviceStatsMonitor.on_train_batchŽ6Ú_start/inactive_split.all.allocated‚36950348 UHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚73070592 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17104896 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 122905600 D;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚5264 C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚4289 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚43507543 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚20643840 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚322 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚262 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 125829120 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚36950280 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚43507545 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚39444698 JBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚665 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 2835126018484 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 122306526 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 5494538240 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17104896 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚320635228651520 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚279499382652928 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 4628521206 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚41348020 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 5368709120 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 5368709120 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 @7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚5264 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 5494538240 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚322 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚320635228651520 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚323476287292928 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 2835084670464 IDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 4654602240 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚320635245756416 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 trainer/global_step‚11749 M@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚59049984 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 4654602240 IDeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4289 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚43507545 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚209 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 3004293287936 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 5494538240 PCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚59049984 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 4552982528 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚321574392717748 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚42873†Î¦Ã'½O YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 125829120 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚39440411 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 122905600 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4289 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚43507545 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚209 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 3004293287936 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 5494538240 PCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚59049984 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 4552982528 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚321574392717748 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚4287 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 2841041536512 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚82952243 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚52426752 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 F?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚60 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 4552982528 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 IBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚68 G>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4876 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚41945088 PIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚66 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚61478912 _step‚966 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚4287 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚27087395 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚223 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 2840999591424 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚9862953 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚282503623514112 KDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚60 IBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚60 _runtime‚3132.819926834 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚323476287292928 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚320635245756416 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚323476228242944 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚321574334330368 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 4517085184 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 2841041536512 KBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚4876 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚262 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚429 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚27087393 PCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚58387380 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚9862887 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚322 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚39440411 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 2840999591424 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚282503696584704 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚279499403296768 BDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚323476228242944 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚36950348 UHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚73070592 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17104896 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 122905600 D;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚5264 C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚4289 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚43507543 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚20643840 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚322 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚262 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 125829120 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚36950280 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚43507545 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚39444698 JBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚665 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 2835126018484 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 122306526 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 5494538240 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17104896 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚320635228651520 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚279499382652928 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 4628521206 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚41348020 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 5368709120 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 5368709120 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 @7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚5264 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 5494538240 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚322 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚320635228651520 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚323476287292928 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 2835084670464 IDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 4654602240 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚320635245756416 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 trainer/global_step‚11749 M@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚59049984 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 4654602240 IDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚320661550989312 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚82959089 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚43511098 EDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 G@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚60 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚323502781649408 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 5494538240 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 IBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚60 trainer/global_step‚11749 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 2841272608256 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚4293 NGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚59 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 NADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚59053056 I@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚4876 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚20643840 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 5368709120 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚43511096 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 2835356591144 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚262 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 B9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚5264 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 2841230660096 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 4552982528 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 >5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5264 E8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚82959089 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚82954794 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚320661568094208 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521206 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚39443698 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚41948160 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚82954794 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 5494538240 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚323502840702464 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17104896 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚279522450669568 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚321600750696516 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚318765452492800 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚322 K>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59053056 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 4552982528 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 122905600 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654602240 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚279522430025728 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚41937920 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚320661568094208 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 122306526 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17104896 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚320661550989312 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚39447991 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚4293 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚36953290 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚27089600 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚209 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 3004534998016 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 2841230660096 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚36953351 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚318765435453440 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 2835315243076 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 125829120 D=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚60 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚61478912 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 5368709120 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 125829120 D6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚322 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚262 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚39443698ÛØš ÇM G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚82959089 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚82954794 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚320661568094208 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521206 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚39443698 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚41948160 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚82954794 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 5494538240 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚323502840702464 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17104896 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚279522450669568 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚321600750696516 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚318765452492800 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚322 K>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59053056 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 4552982528 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 122905600 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654602240 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚279522430025728 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚41937920 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚320661568094208 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 122306526 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17104896 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚320661550989312 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚39447991 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚4293 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚36953290 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚27089600 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚209 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 3004534998016 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 2841230660096 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚36953351 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚318765435453440 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 2835315243076 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 125829120 D=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚60 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚61478912 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 5368709120 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 125829120 D6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚322 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚262 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚39443698 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚282526985667584 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 4517085184 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 5368709120 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 4654602240 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚41948160 SFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚62581760 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 125829120 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚39447991 PDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚320661550989312 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚82959089 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚43511098 EDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 G@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚60 S?ù"ûÓÅ DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚323502781649408 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 5494538240 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 IBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚60 trainer/global_step‚11749 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 2841272608256 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚4293 NGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚59 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 NADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚59053056 I@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚4876 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚20643840 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 5368709120 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚43511096 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 2835356591144 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚262 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 B9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚5264 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 2841230660096 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 4552982528 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 >5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5264 E8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0Ç ¾;Œ2¡- _wandbruntime‚3132  _runtime‚3132IÒµ²t¢î /loss/FRt1/detection_loss‚0.5115890721399149 %loss/total_loss‚2.071608612063498 % _timestamp‚1.7866617844492118e+09 _runtime‚3132.979078023 _step‚968 0loss/FRt1/assignment_loss‚0.5978686331960946 0loss/FRt2/assignment_loss‚0.5104799496114596 0loss/FRt2/detection_loss‚0.45167095711602895 epoch‚29 trainer/global_step‚11749ÁíðNy£ó %loss/total_loss‚2.071608612063498 % _timestamp‚1.7866617844492118e+09 _runtime‚3132.979078023 _step‚968 0loss/FRt1/assignment_loss‚0.5978686331960946 0loss/FRt2/assignment_loss‚0.5104799496114596 0loss/FRt2/detection_loss‚0.45167095711602895 epoch‚29 trainer/global_step‚11749 /loss/FRt1/detection_loss‚0.5115890721399149ÈÎÛa5”¤‚(:‰ š—ùÓÈ÷üà gpu.0.gpu‚0.0 gpu.0.memory‚0, gpu.0.memoryAllocated‚7.0313262939453125) gpu.0.memoryAllocatedBytes‚ 6039863296 gpu.0.temp‚36.0 gpu.0.powerWatts‚211.253' gpu.0.enforcedPowerLimitWatts‚500.0 gpu.0.powerPercent‚42.2506 gpu.0.smClock‚1410 gpu.0.memoryClock‚1593$ gpu.0.correctedMemoryErrors‚1955# gpu.0.uncorrectedMemoryErrors‚0¬)±•¥‚(:Š š—ùÓؽ“ä cpu‚27.190847846467335 disk.dm-0.in‚43669.62890625( disk.nvme0c0n1.out‚58262.07861328125 disk.sda.out‚ 58.05859375 disk.sda2.in‚0 disk.sda2.out‚0& disk.nvme0n1.out‚58262.07861328125 disk.drbd2165.out‚0# proc.memory.rssMB‚ 2260.64453125 network.recv‚ 5.814138e+06 disk.sda3.in‚ 29291.625 disk.sda3.out‚ 58.05859375$ memory_percent‚67.61300563812256 proc.cpu.threads‚17 network.sent‚ 1.5221207e+07) disk./.usagePercent‚32.69211783318172% disk./.usageGB‚487.13618087768555# disk.dm-0.out‚58262.07861328125$ disk.nvme0c0n1.in‚43669.62890625 disk.sda1.out‚0" disk.nvme0n1.in‚43669.62890625 disk.dm-1.in‚ 29291.625 disk.sda.in‚ 29291.625 disk.sda1.in‚0 disk.dm-1.out‚ 58.05859375 disk.drbd2165.in‚0) proc.memory.availableMB‚ 5306.28515625* proc.memory.percent‚13.797879219055176_\×Ï2¦- _wandbruntime‚3145  _runtime‚3145­Oˆ@ȧ _step‚969 & lr-AdamW/pg1‚0.0005598753469002653 & lr-AdamW/pg2‚0.0005598753469002653 trainer/global_step‚11799 % _timestamp‚1.7866617964829204e+09 _runtime‚3145.0129169923/ ꬂ _runtime‚3145.012916992 _step‚969 & lr-AdamW/pg1‚0.0005598753469002653 & lr-AdamW/pg2‚0.0005598753469002653 trainer/global_step‚11799 % _timestamp‚1.7866617964829204e+09ɺðZt2©- _wandbruntime‚3145  _runtime‚3145l(åñ¾'ª¸O PCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚59049984 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚322 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 2852586357248 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 2852628302336 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚9903845 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚262 IBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚60 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 845178880 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚280651751292928 BDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 5494538240 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 D;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚5264 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚39609800 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚262 F?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚60 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚43685193 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 4654602240 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚27197643 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚283668357566976 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚61478912 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚321951362646016 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚43685195 PCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚58387380 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 4628521206 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 4552982528 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 2852586357248 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚52426752 _runtime‚3145.017909089 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚324803990948352 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚20643840 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4289 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚209 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚322895185817776 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 122306526 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 5494538240 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 5368709120 _step‚970 KBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚4876 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚37101488 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 3016533203456 UHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚73070592 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 2846687829092 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚429 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 3016585630208 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚41348020 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 2846646481072 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 122905600 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚4287 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚324803931898368 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17104896 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚322895244205156 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚41945088 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚41945088 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚83294995 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚43685193 JBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚665 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 KDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚60 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚4287 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 4517085184 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚322 @7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚5264 F>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚324803931898368 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 4654602240 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 4552982528 IBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚73 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚39605513 IDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4876 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚321951345541120 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚39605513 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 125829120 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 125829120¡ÛìÃ'«½O M@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚59049984 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 2852628302336 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚223 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 5368709120 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚262 E8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚83290706 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚321951362646016 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚283668284496384 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 5494538240 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 D;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚5264 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚39609800 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚262 F?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚60 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚43685193 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 4654602240 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚27197643 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚283668357566976 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚61478912 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚321951362646016 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚43685195 PCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚58387380 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 4628521206 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 4552982528 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 2852586357248 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚52426752 _runtime‚3145.017909089 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚324803990948352 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚20643840 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4289 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚209 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚322895185817776 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 122306526 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 5494538240 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 5368709120 _step‚970 KBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚4876 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚37101488 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 3016533203456 UHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚73070592 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 2846687829092 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚429 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 3016585630208 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚41348020 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 2846646481072 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 122905600 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚4287 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚324803931898368 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17104896 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚322895244205156 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚41945088 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚41945088 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚83294995 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚43685193 JBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚665 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 KDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚60 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚4287 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 4517085184 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚322 @7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚5264 F>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚324803931898368 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 4654602240 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 4552982528 IBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚73 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚39605513 IDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4876 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚321951345541120 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚39605513 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 125829120 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 125829120 PCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚59049984 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚322 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 2852586357248 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 2852628302336 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚9903845 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚262 IBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚60 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 845178880 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚280651751292928 BDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654602240 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17104896 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚322921660571352 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚322 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚83301841 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚37104548 SFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚64678912 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚44035072 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 5494538240 _step‚971 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 2852817425920 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 3016782592512 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 122306526 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 125829120 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 125829120 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚39613093 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 2852859374080 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚39613093 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚283691581258240 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚322 $ _timestamp‚1.786661796646726e+09 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚39608800 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 122905600 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚429 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚43688746 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚83301841 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 5494538240 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 2852817425920 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚43688748 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚320074725130240 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚4295 >5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5264 E8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 2846918401752 _runtime‚3145.177015757 REDeviceStatsMonitor.on_train_batch_end/active_bytþa¡â es.small_pool.current‚41948160 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 122905600 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚324830544357888 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚83297546 I@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚4876 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 845178880 D=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚60 D6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚322 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚320074742169600 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚41948160 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚61478912 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 IBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚60 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚324830485304832 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚283691645937152 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521206 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 125829120 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚322 K>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59053056 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 4552982528 H@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚665 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚39608800 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚209 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚43688746 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚37104485 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚280674819309568 G@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚60 2YÌ&®ÆM ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚320074742169600 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚41948160 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚61478912 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 IBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚60 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚324830485304832 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚283691645937152 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521206 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 125829120 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚322 K>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59053056 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 4552982528 H@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚665 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚39608800 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚209 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚43688746 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚37104485 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚280674819309568 G@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚60 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17104896 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚324830544357888 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚321977667878912 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚9904637 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 2846877053684 trainer/global_step‚11799 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 NADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚58387428 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚41348068 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 5368709120 NGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚61 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚262 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚83297546 PDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654602240 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17104896 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚322921660571352 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚322 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚83301841 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚37104548 SFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚64678912 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚44035072 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 5494538240 _step‚971 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 2852817425920 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 3016782592512 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 122306526 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 125829120 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 125829120 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚39613093 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 2852859374080 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚39613093 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚283691581258240 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚322 $ _timestamp‚1.786661796646726e+09 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚39608800 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 122905600 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚429 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚43688746 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚83301841 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 5494538240 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 2852817425920 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚43688748 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚320074725130240 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚4295 >5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5264 E8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 2846918401752 _runtime‚3145.177015757 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚41948160 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 122905600 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚324830544357888 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚83297546 I@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚4876 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 845178880 D=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚60 D6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚322ËRhá2¯- _wandbruntime‚3145  _runtime‚3145ßþ=t°î 0loss/FRt1/assignment_loss‚0.5622692463422894 /loss/FRt1/detection_loss‚0.5237804024343419 /loss/FRt2/detection_loss‚0.4926035351819868 &loss/total_loss‚2.0958106038736384 _step‚972 0loss/FRt2/assignment_loss‚0.5171574199150206 epoch‚29 trainer/global_step‚11799 % _timestamp‚1.7866617966477926e+09 _runtime‚3145.177657587,þ9ÿy±ó 0loss/FRt1/assignment_loss‚0.5622692463422894 /loss/FRt1/detection_loss‚0.5237804024343419 /loss/FRt2/detection_loss‚0.4926035351819868 &loss/total_loss‚2.0958106038736384 _step‚972 0loss/FRt2/assignment_loss‚0.5171574199150206 epoch‚29 trainer/global_step‚11799 % _timestamp‚1.7866617966477926e+09 _runtime‚3145.177657587Ìm{&Æ–²‚(:‹ ©—ùÓ¹ˆùà gpu.0.gpu‚63.0 gpu.0.memory‚12, gpu.0.memoryAllocated‚7.0313262939453125) gpu.0.memoryAllocatedBytes‚ 6039863296 gpu.0.temp‚37.0 gpu.0.powerWatts‚177.171' gpu.0.enforcedPowerLimitWatts‚500.0 gpu.0.powerPercent‚35.4342 gpu.0.smClock‚1410 gpu.0.memoryClock‚1593$ gpu.0.correctedMemoryErrors‚1955# gpu.0.uncorrectedMemoryErrors‚0fl“³‚(:ˆ ©—ùÓã’Üâ disk.dm-0.in‚43669.62890625# disk.dm-0.out‚58267.68115234375 disk.sda3.out‚ 58.66015625$ memory_percent‚67.61434078216553# proc.memory.rssMB‚ 2260.64453125 cpu‚27.19068083471763 disk.sda.in‚ 29291.625 disk.sda1.out‚0 disk.sda2.in‚0 disk.sda2.out‚0 disk.drbd2165.in‚0 disk.sda3.in‚ 29291.625( disk.nvme0c0n1.out‚58267.68115234375* proc.memory.percent‚13.797879219055176 network.recv‚ 5.832437e+06$ disk./.usageGB‚487.0069046020508 disk.sda.out‚ 58.66015625& disk.nvme0n1.out‚58267.68115234375 disk.dm-1.in‚ 29291.625 disk.drbd2165.out‚0) proc.memory.availableMB‚ 5306.06640625 disk.sda1.in‚0" disk.nvme0n1.in‚43669.62890625 disk.dm-1.out‚ 58.66015625$ disk.nvme0c0n1.in‚43669.62890625 proc.cpu.threads‚17 network.sent‚ 1.530376e+07* disk./.usagePercent‚32.683441993853855:}¨02´- _wandbruntime‚3157  _runtime‚3157eÐÚ;ȵ & lr-AdamW/pg1‚0.0005540671510581017 & lr-AdamW/pg2‚0.0005540671510581017 trainer/global_step‚11849 % _timestamp‚1.7866618087083879e+09 _runtime‚3157.238413269 _step‚973@í*Ðꦂ _runtime‚3157.238413269 _step‚973 & lr-AdamW/pg1‚0.0005540671510581017 & lr-AdamW/pg2‚0.0005540671510581017 trainer/global_step‚11849 % _timestamp‚1.7866618087083879e+09ÍN 1z2·- _wandbruntime‚3157  _runtime‚3157ñEP‚¾'¸¸O _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 3028842421248 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 4517085184 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚4287 C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚4289 G>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4876 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚281804140576768 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚9944385 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚324216095692564 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚43862845 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚27307893 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 4654602240 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17104896 D;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚5264 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚4287 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚323267462430720 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚83637747 KBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚4876 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚284832962354176 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 2858208291680 E8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚83633458 BDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚326131635553792 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚324216037305184 F>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 2858249639700 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 5368709120 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚322 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚9944445 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 122306526 _step‚974 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 122905600 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚20643840 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚61478912 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 5368709120 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 125829120 @7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚5264 M@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚59049984 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚284833024939008 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 845178880 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4289 JBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚665 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 5368709120 % _timestamp‚1.7866618087093453e+09 _runtime‚3157.242883812 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚39770615 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚281804119932928 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚322 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚321357846052864 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚43862843 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 125829120 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 2864173123072 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚43862843 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 831258624 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 5494538240 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚262 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 122905600 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 PCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚58387380 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 4628521206 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚41348020 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚223 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚323267462430720 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 F?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚60 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 3028884362240 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚37252340 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚326131635553792 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚41945088 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚39770615 UHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚62584832 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚322 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚322 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚262 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 2864215068160 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚37252278 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚323267479535616 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 125829120 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚262 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 IDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4876 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚281804140576768 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚9944385 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚324216095692564 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚43862845 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚27307893 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 4654602240 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17104896 D;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚5264 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚4287 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚323267462430720 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚83637747 KBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚4876 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚284832962354176 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 2858208291680 E8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚83633458 BDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚326131635553792 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚324216037305184 F>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 2858249639700 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 5368709120 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚322 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚9944445 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 122306526 _step‚974 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 122905600 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚20643840 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚61478912 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 5368709120 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 125829120 @7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚5264 M@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚59049984 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚284833024939008 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 845178880 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4289 JBDeviceStatsMonitor.on_train_batch÷ Ë _start/allocation.large_pool.peak‚665 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 5368709120 % _timestamp‚1.7866618087093453e+09 _runtime‚3157.242883812 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚39770615 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚281804119932928 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚322 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚321357846052864 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚43862843 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 125829120 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 2864173123072 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚43862843 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 831258624 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 5494538240 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚262 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 122905600 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 PCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚58387380 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 4628521206 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚41348020 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚223 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚323267462430720 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 F?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚60 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 3028884362240 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚37252340 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚326131635553792 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚41945088 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚39770615 UHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚62584832 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚322 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚322 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚262ÎÌþ}2º- _wandbruntime‚3157  _runtime‚3157$‹ sÇ&»ÁM UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 2864404191744 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚323293801873408 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 2864446139904 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚284856248794112 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 5494538240 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚262 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 4552982528 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 122905600 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 3029081488384 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 2858438864292 D=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚60 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17104896 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚39778195 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚20643840 _step‚975 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17104896 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚324242512058760 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚321384014807040 trainer/global_step‚11849 $ _timestamp‚1.786661808874608e+09 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚323293784768512 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚39778195 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚321384031846400 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚43866398 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 4654602240 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚326158248013312 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚4295 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚4293 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚9945238 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 831258624 NADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚59053056 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 SFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚70970368 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 H@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚665 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚37255259 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 125829120 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚322 NADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚58387428 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚83644593 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚43866396 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚262 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚326158188960256 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 5368709120 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 2858480212360 G@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚60 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚326158248013312 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚43866396 E6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚322 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚83644593 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚27310098 I@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚4876 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 125829120 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚41348068 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚323293801873408 EDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59053056 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654602240 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 5494538240 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 125829120 P5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5264 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚39773902 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚281827167305728 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚209 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚322 BDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521206 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 2864404191744 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚429 D8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 4517085184 B9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚5264 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚43866398 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚324242453671332 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚83640298 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚41948160 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 5368709120çååÖÌ&¼ÆM A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚4295 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚4293 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚9945238 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 831258624 NADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚59053056 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 SFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚70970368 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 H@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚665 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚37255259 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 125829120 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚322 NADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚58387428 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚83644593 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚43866396 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚262 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚326158188960256 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 5368709120 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 2858480212360 G@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚60 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚326158248013312 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚43866396 E6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚322 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚83644593 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚27310098 I@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚4876 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 125829120 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚41348068 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚323293801873408 EDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59053056 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654602240 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 5494538240 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 125829120 P5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5264 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚39773902 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚281827167305728 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚209 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚322 BDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521206 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 2864404191744 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚429 D8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 4517085184 B9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚5264 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚43866398 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚324242453671332 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚83640298 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚41948160 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 5368709120 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 2864404191744 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚323293801873408 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 2864446139904 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚284856248794112 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 5494538240 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚262 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 4552982528 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 122905600 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 3029081488384 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 2858438864292 D=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚60 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17104896 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚39778195 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚20643840 _step‚975 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17104896 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚324242512058760 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚321384014807040 trainer/global_step‚11849 $ _timestamp‚1.786661808874608e+09 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚323293784768512 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚39778195 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚321384031846400 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚43866398 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 4654602240 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚326158248013312Ï7?2½- _wandbruntime‚3157  _runtime‚3157xï/It¾î _runtime‚3157.405834271 0loss/FRt1/assignment_loss‚0.5851117552741126 0loss/FRt2/assignment_loss‚0.5273778893539158 0loss/FRt1/detection_loss‚0.47183704070273486 /loss/FRt2/detection_loss‚0.4757033705963177 %loss/total_loss‚2.060030055927081 epoch‚29 _step‚976 trainer/global_step‚11849 % _timestamp‚1.7866618088760395e+09ùE±Py¿ó epoch‚29 _step‚976 trainer/global_step‚11849 % _timestamp‚1.7866618088760395e+09 _runtime‚3157.405834271 0loss/FRt1/assignment_loss‚0.5851117552741126 0loss/FRt2/assignment_loss‚0.5273778893539158 0loss/FRt1/detection_loss‚0.47183704070273486 /loss/FRt2/detection_loss‚0.4757033705963177 %loss/total_loss‚2.060030055927081Ð_ªŒ”À‚(:‰ ¸—ùÓà¡šá gpu.0.gpu‚27.0 gpu.0.memory‚6, gpu.0.memoryAllocated‚7.0313262939453125) gpu.0.memoryAllocatedBytes‚ 6039863296 gpu.0.temp‚36.0 gpu.0.powerWatts‚214.475' gpu.0.enforcedPowerLimitWatts‚500.0 gpu.0.powerPercent‚42.895 gpu.0.smClock‚1410 gpu.0.memoryClock‚1593$ gpu.0.correctedMemoryErrors‚1955# gpu.0.uncorrectedMemoryErrors‚0Ç Y€Á‚(:õ ¸—ùÓ¬¬õâ disk.drbd2165.out‚0# memory_percent‚67.6142692565918' proc.memory.availableMB‚ 5306.078125 cpu‚27.190330660266905% disk./.usageGB‚487.10972595214844 disk.sda1.out‚0 disk.nvme0n1.in‚ 43669.8125 disk.dm-1.in‚ 29291.625 disk.drbd2165.in‚0# disk.dm-0.out‚58332.83837890625 disk.sda3.in‚ 29291.625 disk.sda3.out‚ 58.984375 network.sent‚ 1.5379106e+07 network.recv‚ 5.865273e+06 disk.sda.in‚ 29291.625 disk.sda2.out‚0& disk.nvme0n1.out‚58332.83837890625 disk.dm-1.out‚ 58.984375 disk.nvme0c0n1.in‚ 43669.8125( disk.nvme0c0n1.out‚58332.83837890625 disk.sda.out‚ 58.984375 disk.sda1.in‚0 disk.dm-0.in‚ 43669.8125# proc.memory.rssMB‚ 2260.64453125* proc.memory.percent‚13.797879219055176 proc.cpu.threads‚17) disk./.usagePercent‚32.69034242093177 disk.sda2.in‚0c=4v2Â- _wandbruntime‚3169  _runtime‚3169¨%ZÈàtrainer/global_step‚11899 % _timestamp‚1.7866618213275185e+09 _runtime‚3169.857478857 _step‚977 & lr-AdamW/pg1‚0.0005482715396166953 & lr-AdamW/pg2‚0.0005482715396166953¢%ÍÄÇ & lr-AdamW/pg2‚0.0005482715396166953 trainer/global_step‚11899 % _timestamp‚1.7866618213275185e+09 _runtime‚3169.857478857 _step‚977 & lr-AdamW/pg1‚0.0005482715396166953јü^l2Å- _wandbruntime‚3169  _runtime‚3169Å¿% âƸO L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚39935717 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 4654602240 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚429 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚27418145 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 _runtime‚3169.861910234 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚327459398259200 M@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚59049984 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚37402906 IBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚60 PCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚59049984 KBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚4876 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚282956488572928 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 KDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚60 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚324583579320320 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 2875759888896 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚322 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚322667118690304 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚324583596425216 BDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 122306526 trainer/global_step‚11899 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 122905600 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚209 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚27418143 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 4628521206 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 5368709120 % _timestamp‚1.7866618213283744e+09 IDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4876 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 2875801833984 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 122905600 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 @7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚5264 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚44040493 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚39940004 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚9984763 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 5368709120 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚327459339209216 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚39935717 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚37402972 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 5368709120 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚322 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚262 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚44040493 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 3041150605312 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚322667135729664 E8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚83976210 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚4287 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 2875801833984 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚322 F?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚60 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚44040495 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚20643840 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚48232448 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚39940004 ^JDev†¢Ø³Ü iceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚285997659822080 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 125829120 _step‚978 D;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚5264 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚44040495 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 3041102372864 PCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚58387380 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 125829120 F>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚327459339209216 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17104896 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4289 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚9984827 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚262 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚4287 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚41348020 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 2869770102288 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚322 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚262 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 2869811450308 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 IDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 122306526 trainer/global_step‚11899 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 122905600 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚209 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚27418143 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 4628521206 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 5368709120 % _timestamp‚1.7866618213283744e+09 IDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4876 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 2875801833984 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 122905600 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 @7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚5264 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚44040493 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚39940004 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚9984763 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 5368709120 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚327459339209216 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚39935717 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚37402972 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 5368709120 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚322 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚262 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚44040493 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 3041150605312 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚322667135729664 E8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚83976210 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚4287 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 2875801833984 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚322 F?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚60 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚44040495 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚20643840 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚48232448 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚39940004 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚285997659822080 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 125829120 _step‚978 D;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚5264 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚44040495 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 3041102372864 PCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚58387380 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 125829120 F>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚327459339209216 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17104896 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4289 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚9984827 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚262 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚4287 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚41348020 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 2869770102288 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚322 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚262 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 2869811450308 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 IDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚41948160 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚83987345 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚429 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 3041391111168 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 845178880 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚322693304483840 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 5494538240 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚44044048 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚44044046 D6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚322 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 BDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚37405921 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚223 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚20643840 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚39939004 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 PDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654602240 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚27420348 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 2870042022968 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 5368709120 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚4293 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚39939004 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚325563363546168 >5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5264 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚282979556589568 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 122306526 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 K>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59053056 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 2876032905728 B9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚5264 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚39943297 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 3041351270400 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚324609918763008 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚324609918763008 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚324609901658112 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚327485951668736 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚209 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚286020947700736 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚282979535945728 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚262 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚4293 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 4654602240 NGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚63 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚286020887216128 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚41348068 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17104896 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚44044048 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚9985573 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚61478912 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚325563305158740 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 4517085184 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 125829120 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚322 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚83983050 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚37405986 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 2870000674900 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 5494538240 NADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚59053056 G@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚65 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚322 D=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚60 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚327485951668736 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 SFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚60484608 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 5494538240 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚4295 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 4552982528 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚39840768 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521206 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 125829120ö>þïD%ÊÇM PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 122905600 EDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚41948160 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚83987345 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚429 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 3041391111168 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 845178880 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚322693304483840 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 5494538240 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚44044048 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚44044046 D6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚322 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 BDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚37405921 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚223 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚20643840 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚39939004 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 PDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654602240 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚27420348 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 2870042022968 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 5368709120 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚4293 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚39939004 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚325563363546168 >5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5264 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚282979556589568 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 122306526 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 K>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59053056 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 2876032905728 B9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚5264 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚39943297 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 3041351270400 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚324609918763008 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚324609918763008 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚324609901658112 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚327485951668736 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚209 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚286020947700736 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚282979535945728 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚262 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚4293 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 4654602240 NGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚63 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚286020887216128 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚41348068 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17104896 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚44044048 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚9985573 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚61478912 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚325563305158740 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 4517085184 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 125829120 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚322 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚83983050 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚37405986 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 2870000674900 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 5494538240 NADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚59053056 G@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚65 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚322 D=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚60 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚327485951668736 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 SFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚60484608 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 5494538240 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚4295 TEDeviceStatsMonitor.on_train_batch_end/allocated_byte8Êɉs.large_pool.peak‚ 4552982528 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚39840768 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521206 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 125829120 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 4552982528 _step‚979Óá˜*_2Ë- _wandbruntime‚3170  _runtime‚3170|âïŽsÌí /loss/FRt1/detection_loss‚0.5127501085810848 /loss/FRt2/detection_loss‚0.4716538768274395 trainer/global_step‚11899 _runtime‚3170.023968246 _step‚980 0loss/FRt1/assignment_loss‚0.6547114476562182 /loss/FRt2/assignment_loss‚0.572212364469511 &loss/total_loss‚2.2113277975342536 epoch‚29 % _timestamp‚1.7866618214941502e+09ÚéëqxÍò /loss/FRt1/detection_loss‚0.5127501085810848 /loss/FRt2/detection_loss‚0.4716538768274395 trainer/global_step‚11899 _runtime‚3170.023968246 _step‚980 0loss/FRt1/assignment_loss‚0.6547114476562182 /loss/FRt2/assignment_loss‚0.572212364469511 &loss/total_loss‚2.2113277975342536 epoch‚29 % _timestamp‚1.7866618214941502e+09Ô ¿í–΂(:‹ Ç—ùÓöìûà gpu.0.gpu‚62.0 gpu.0.memory‚12, gpu.0.memoryAllocated‚7.0313262939453125) gpu.0.memoryAllocatedBytes‚ 6039863296 gpu.0.temp‚37.0 gpu.0.powerWatts‚178.473' gpu.0.enforcedPowerLimitWatts‚500.0 gpu.0.powerPercent‚35.6946 gpu.0.smClock‚1410 gpu.0.memoryClock‚1593$ gpu.0.correctedMemoryErrors‚1955# gpu.0.uncorrectedMemoryErrors‚04uó‹Ï‚(:€ Ç—ùÓ›êÏâ disk.dm-1.in‚ 29291.625! disk.dm-0.out‚58337.033203125& disk.nvme0c0n1.out‚58337.033203125' proc.memory.availableMB‚ 5306.359375 network.sent‚ 1.5454239e+07 network.recv‚ 5.898035e+06$ disk./.usageGB‚487.1099166870117 disk.sda1.in‚0 disk.sda2.out‚0 disk.dm-1.out‚ 59.37890625 disk.dm-0.in‚43708.29296875$ memory_percent‚67.61255264282227) disk./.usagePercent‚32.69035522130848 disk.sda2.in‚0 disk.drbd2165.in‚0 disk.drbd2165.out‚0 disk.sda3.in‚ 29291.625$ disk.nvme0c0n1.in‚43708.29296875# proc.memory.rssMB‚ 2260.64453125* proc.memory.percent‚13.797879219055176 disk.sda.in‚ 29291.625 disk.sda3.out‚ 59.37890625 cpu‚27.18983696452851 proc.cpu.threads‚17 disk.sda.out‚ 59.37890625 disk.sda1.out‚0" disk.nvme0n1.in‚43708.29296875$ disk.nvme0n1.out‚58337.033203125·Ú¦Ã2Ð- _wandbruntime‚3182  _runtime‚3182(¾½ ÈÑ & lr-AdamW/pg1‚0.0005424888848173389 & lr-AdamW/pg2‚0.0005424888848173389 trainer/global_step‚11949 % _timestamp‚1.7866618337503266e+09 _runtime‚3182.282888129 _step‚981D“ •ÍÒÇ % _timestamp‚1.7866618337503266e+09 _runtime‚3182.282888129 _step‚981 & lr-AdamW/pg1‚0.0005424888848173389 & lr-AdamW/pg2‚0.0005424888848173389 trainer/global_step‚11949Õí?‰2Ó- _wandbruntime‚3182  _runtime‚31828ZoýÀ'ÔºO HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 trainer/global_step‚11949 _step‚982 E8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚84318962 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚48232448 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚322 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚44218145 PCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚59049984 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 2887346654720 PIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚76 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚326857798667380 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚323976425406464 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 5494538240 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 5368709120 IDeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4289 IDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚328787042864640 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚429 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚284108877856768 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 5368709120 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 831258624 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 4628521206 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚322 _runtime‚3182.282967579 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 125829120 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚325899696209920 M@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚59049984 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚10025123 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 4654602240 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚41945088 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 4517085184 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚41348020 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚325899696209920 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚44218145 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 2881373260916 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚262 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 4552982528 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 5494538240 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 5368709120 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 125829120 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚27528395 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 2887346654720 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚328787101914624 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 3053414617088 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚262 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 % _timestamp‚1.7866618337509751e+09 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚328787042864640 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚37553594 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 D;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚5264 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚40105106 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚10025199 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚322 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚323976408367104 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚4289 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚40100819 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚40100819 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚322 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 122905600 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 845178880 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚44218143 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚284108857212928 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚37553516 G>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4876 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚325899713314816 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17104896 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 5494538240 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 125829120 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚41945088 UHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚68876288 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 2887388599808 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 122905600 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚325899713314816 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚287162271830016 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 122306526 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 4654602240 F>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 2887388599808 IBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚78 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚287162340706304 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚326857740280000 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚262 IBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚60 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚44218143 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚223 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚209 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 3053462849536 BDeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4289 IDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚328787042864640 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚429 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚284108877856768 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 5368709120 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 831258624 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 4628521206 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚322 _runtime‚3182.282967579 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 125829120 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚325899696209920 M@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚59049984 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚10025123 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 4654602240 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚41945088 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 4517085184 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚41348020 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚325899696209920 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚44218145 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 2881373260916 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚262 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 4552982528 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 5494538240 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 5368709120 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 125829120 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚27528395 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 2887346654720 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚328787101914624 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 3053414617088 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚262 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 % _timestamp‚1.7866618337509751e+09 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚328787042864640 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚37553594 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 D;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚5264 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚40105106 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚10025199 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚322 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚323976408367104 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚4289 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚40100819 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚40100819 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚322 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 122905600 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 845178880 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚44218143 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚284108857212928 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚37553516 G>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4876 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚325899713314816 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17104896 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 5494538240 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 125829120 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚41945088 UHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚68876288 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 2887388599808 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 122905600 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚325899713314816 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚287162271830016 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 122306526 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 4654602240 F>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 2887388599808 IBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚78 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚287162340706304 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚326857740280000 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚262 IBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚60 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚44218143 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚223 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚209 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 3053462849536 BDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚37556538 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 % _timestamp‚1.7866618339123847e+09 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚4293 K>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59053056 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17104896 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚287185566855168 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 4552982528 G@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚65 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚44221696 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 845178880 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚262 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654602240 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 3053702110208 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚322 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚325926018547712 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 125829120 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚84330097 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 2887619671552 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚20643840 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 125829120 IBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚60 NADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚59053056 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚41948160 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚40104106 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521206 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚324002594160640 PDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚84330097 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 5494538240 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚61478912 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 2881603833576 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 2881562485508 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚262 D=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚60 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 2887619671552 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚27530600 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚326884156646148 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 122905600 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚44221698 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 5494538240 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚44221698 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 D5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5264 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 2887577723392 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 122905600 E6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚322 B9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚5264 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚37556603 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 3053662269440 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚322 QBDeviceStatsMonitor.on_train_batcP•"£h_end/reserved_bytes.all.allocated‚ 5494538240 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 4654602240 NGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚63 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚287185627339776 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 4517085184 _step‚983 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0ÔdŽÏ&ØÉM >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 2881562485508 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚262 D=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚60 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 2887619671552 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚27530600 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚326884156646148 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 122905600 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚44221698 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 5494538240 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚44221698 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 D5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5264 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 2887577723392 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 122905600 E6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚322 B9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚5264 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚37556603 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 3053662269440 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚322 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 5494538240 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 4654602240 NGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚63 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚287185627339776 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 4517085184 _step‚983 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 125829120 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚209 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚326884215033576 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 5368709120 I@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚4876 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚37556538 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 % _timestamp‚1.7866618339123847e+09 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚4293 K>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59053056 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17104896 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚287185566855168 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 4552982528 G@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚65 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚44221696 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 845178880 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚262 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654602240 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 3053702110208 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚322 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚325926018547712 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 125829120 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚84330097 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 2887619671552 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚20643840 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 125829120 IBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚60 NADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚59053056 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚41948160 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚40104106 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521206 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚324002594160640 PDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚84330097 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 5494538240 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚61478912 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 2881603833576 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2׺˜í2Ù- _wandbruntime‚3182  _runtime‚3182]OtÚî /loss/FRt1/detection_loss‚0.5041744766767725 epoch‚29 _runtime‚3182.443280927 0loss/FRt2/assignment_loss‚0.5583086802369079 /loss/FRt2/detection_loss‚0.4756657995010501 &loss/total_loss‚2.1883568297997833 trainer/global_step‚11949 % _timestamp‚1.7866618339134657e+09 _step‚984 0loss/FRt1/assignment_loss‚0.6502078733850527•]€yyÛó epoch‚29 _runtime‚3182.443280927 0loss/FRt2/assignment_loss‚0.5583086802369079 /loss/FRt2/detection_loss‚0.4756657995010501 &loss/total_loss‚2.1883568297997833 trainer/global_step‚11949 % _timestamp‚1.7866618339134657e+09 _step‚984 0loss/FRt1/assignment_loss‚0.6502078733850527 /loss/FRt1/detection_loss‚0.5041744766767725Ø‹3Ù2Ü- _wandbruntime‚3194  _runtime‚3194׿´fÆÝÀ $ _timestamp‚1.786661846009688e+09 _runtime‚ 3194.53976717 _step‚985 & lr-AdamW/pg1‚0.0005367195580691447 & lr-AdamW/pg2‚0.0005367195580691447 trainer/global_step‚11999“¼vËÞÅ & lr-AdamW/pg1‚0.0005367195580691447 & lr-AdamW/pg2‚0.0005367195580691447 trainer/global_step‚11999 $ _timestamp‚1.786661846009688e+09 _runtime‚ 3194.53976717 _step‚985Ùmüª“2ß- _wandbruntime‚3194  _runtime‚3194ÑþÂÀ'àºO YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚61478912 PCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚58387380 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 122306526 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 5494538240 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚40265921 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚328178650154788 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 2898933420544 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚223 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚20643840 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 3065724392960 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 2892935071524 JBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚665 BDeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4289 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 % _timestamp‚1.7866618460106528e+09 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚325285698043904 C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚4289 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚262 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚44395795 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚285261225852928 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 4628521206 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 4654602240 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 F>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 4552982528 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 2898933420544 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 5494538240 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 5368709120 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 2892893723504 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 125829120 F?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚60 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚40270208 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚330114805570048 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚41945088 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 125829120 IDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚330114746520064 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17104896 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚330114746520064 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚27638645 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚209 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 G>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4876 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚327215830204416 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 2898975365632 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚288326970889728 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚322 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚327215813099520 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚285261246496768 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 125829120 trainer/global_step‚11999 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 122905600 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚4287 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚41348020 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚322ŒÊ21Å'á¿O UHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚60487680 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 4517085184 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 5368709120 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 4552982528 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4289 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 % _timestamp‚1.7866618460106528e+09 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚325285698043904 C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚4289 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚262 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚44395795 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚285261225852928 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 4628521206 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 4654602240 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 F>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 4552982528 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 2898933420544 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 5494538240 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 5368709120 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 2892893723504 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 125829120 F?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚60 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚40270208 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚330114805570048 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚41945088 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 125829120 IDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚330114746520064 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17104896 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚330114746520064 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚27638645 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚209 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 G>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4876 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚327215830204416 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 2898975365632 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚288326970889728 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚322 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚327215813099520 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚285261246496768 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 125829120 trainer/global_step‚11999 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 122905600 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚4287 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚41348020 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚322 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚61478912 PCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚58387380 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 122306526 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 5494538240 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚40265921 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚328178650154788 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 2898933420544 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚223 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚20643840 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 3065724392960 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 2892935071524 JBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚665 B5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5264 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚44399348 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚84668554 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.allg^@«b%.freed‚288350206803456 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚37743616 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚61478912 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 5368709120 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚328205066520984 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚328205008133556 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚322 _runtime‚3194.704070705 _step‚987 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 122905600 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚325311883837440 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚330141358979584 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚327242152542208 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 122905600 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 831258624 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 3065971321344 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚40273501 SFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚58387456 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521206 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚330141358979584 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚27640848 NADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚59053056 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654602240 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚84672849 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚285284273225728 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚37706872 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 845178880 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 4552982528 B9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚5264 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚40269208 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚322 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 4517085184 DDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 IBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚60 trainer/global_step‚11999 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17104896 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 2899206437376 NGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚59 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚325311900876800 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 2893124296116 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 122306526 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚44399346 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚262 I@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚4876 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 125829120 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚327242152542208 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚288350265190912 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚44399346 EDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59053056 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚27640850 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚322 PDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚37706811 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 2893165644184 G@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚60 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚41948160 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚330141299926528 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚262 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 125829120 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚4293 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 B8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 5494538240 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚06­ÒÏ&äÉM D=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚60 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 125829120 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 5494538240 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 E5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5264 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚44399348 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚84668554 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚288350206803456 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚37743616 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚61478912 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 5368709120 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚328205066520984 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚328205008133556 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚322 _runtime‚3194.704070705 _step‚987 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 122905600 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚325311883837440 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚330141358979584 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚327242152542208 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 122905600 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 831258624 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 3065971321344 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚40273501 SFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚58387456 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521206 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚330141358979584 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚27640848 NADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚59053056 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654602240 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚84672849 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚285284273225728 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚37706872 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 845178880 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 4552982528 B9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚5264 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚40269208 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚322 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 4517085184 DDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 IBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚60 trainer/global_step‚11999 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17104896 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 2899206437376 NGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚59 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚325311900876800 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 2893124296116 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 122306526 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚44399346 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚262 I@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚4876 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 125829120 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚327242152542208 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚288350265190912 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚44399346 EDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59053056 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚27640850 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚322 PDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚37706811 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 2893165644184 G@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚60 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚41948160 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚330141299926528 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚262 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 125829120 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚4293 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 BDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚331837514166784 IDeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4289 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 4517085184 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚322 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚44634051 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 2914587912704 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚40457502 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚37904270 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 831258624 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 125829120 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚328922884308992 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17104896 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚10114692 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚286712864800768 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 5494538240 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 4654602240 JBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚665 UHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚64681984 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚262 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚262 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 2908521506218 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 125829120 IDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚331837455116800 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 G>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4876 YEDeviceçMmÕ StatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚328922867204096 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚289794924744704 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 F>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 2914629857792 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚4287 KBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚4876 BDeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4289 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 4517085184 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚322 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚44634051 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 2914587912704 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚40457502 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚37904270 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 831258624 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 125829120 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚328922884308992 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17104896 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚10114692 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚286712864800768 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 5494538240 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 4654602240 JBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚665 UHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚64681984 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚262 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚262 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 2908521506218 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 125829120 IDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚331837455116800 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 G>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4876 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚328922867204096 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚289794924744704 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 F>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 2914629857792 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚4287 KBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚4876 BDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚331837514166784 IDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521206 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚322 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 2914818981376 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 3082349698048 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 125829120 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 122905600 EDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 5368709120 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚41948160 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 831258624 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 125829120 >5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5264 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚44637602 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 BDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59053056 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 EDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚40460795 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚429 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚262 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚40460795 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚40456502 PDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚85098399 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚37907213 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚10115501 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚4293 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 845178880 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚322 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚41348068 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚322 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚262 % _timestamp‚1.7866618646339147e+09 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚4295 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 2914860929536 B9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚5264 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚223 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚20643840 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚286735912173568 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 4654602240 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚328949189541888 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 4552982528 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚85094104 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 125829120 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚331864067576320 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17104896 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚328949206646784 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 2914860929536 NADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚58387428 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚331864008523264 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654602240 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚209 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 4517085184 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 2914818981376 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚37907277 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0t.FzG#÷ÉM REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17104896 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚328949206646784 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 2914860929536 NADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚58387428 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚331864008523264 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654602240 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚209 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 4517085184 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 2914818981376 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚37907277 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 2908793426898 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 trainer/global_step‚12049 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚328949206646784 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚27791714 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚329910825084910 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚327002090045440 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚327002073006080 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 122306526 IBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚60 NGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚62 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521206 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚322 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 2914818981376 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 3082349698048 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 125829120 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 122905600 EDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 5368709120 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚41948160 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 831258624 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 125829120 >5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5264 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚44637602 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 BDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59053056 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 EDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚40460795 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚429 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚262 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚40460795 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚40456502 PDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚85098399 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚37907213 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚10115501 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚4293 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 845178880 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚322 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚41348068 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚322 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚262 % _timestamp‚1.7866618646339147e+09 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚4295 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 2914860929536 B9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚5264 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚223 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚20643840 _KDeviceStatsMonitor.on_train_batch_end/inactiž«ê ˆve_split_bytes.large_pool.freed‚286735912173568 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 4654602240 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚328949189541888 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 4552982528 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚85094104 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 125829120 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚331864067576320 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0àz ûÊ2ø- _wandbruntime‚3213  _runtime‚3213¤’Ttùî 0loss/FRt2/assignment_loss‚0.5596643803605025 0loss/FRt2/detection_loss‚0.48403868207652867 epoch‚30 trainer/global_step‚12049 % _timestamp‚1.7866618646350706e+09 _step‚993 0loss/FRt1/assignment_loss‚0.6839029713301409 /loss/FRt1/detection_loss‚0.5005576457882905 %loss/total_loss‚2.228163679555463 _runtime‚3213.164905181lÛ’8yúó _step‚993 0loss/FRt1/assignment_loss‚0.6839029713301409 /loss/FRt1/detection_loss‚0.5005576457882905 %loss/total_loss‚2.228163679555463 _runtime‚3213.164905181 0loss/FRt2/assignment_loss‚0.5596643803605025 0loss/FRt2/detection_loss‚0.48403868207652867 epoch‚30 trainer/global_step‚12049 % _timestamp‚1.7866618646350706e+09áÏ×êß–û‚(:‹ ô—ùÓ䜜á gpu.0.gpu‚52.0 gpu.0.memory‚10, gpu.0.memoryAllocated‚7.0313262939453125) gpu.0.memoryAllocatedBytes‚ 6039863296 gpu.0.temp‚35.0 gpu.0.powerWatts‚119.998' gpu.0.enforcedPowerLimitWatts‚500.0 gpu.0.powerPercent‚23.9996 gpu.0.smClock‚1410 gpu.0.memoryClock‚1593$ gpu.0.correctedMemoryErrors‚1955# gpu.0.uncorrectedMemoryErrors‚0ÙKš“ü‚(:ˆ ô—ùÓìÔþâ$ disk.nvme0n1.out‚58530.431640625* proc.memory.percent‚14.273977279663086 network.recv‚ 5.970945e+06 disk.sda.in‚29326.33203125" disk.nvme0n1.in‚43710.62890625 disk.dm-1.in‚29326.33203125 disk.dm-1.out‚ 60.171875! disk.dm-0.out‚58530.431640625# memory_percent‚68.0915355682373 disk.sda2.in‚0% disk./.usageGB‚487.17058181762695 disk.sda3.in‚29326.33203125 disk.sda.out‚ 60.171875 disk.sda1.in‚0 disk.sda1.out‚0 disk.drbd2165.in‚0 disk.drbd2165.out‚0 network.sent‚ 1.5657902e+07$ disk.nvme0c0n1.in‚43710.62890625 disk.sda2.out‚0 disk.dm-0.in‚43710.62890625( proc.memory.availableMB‚ 5227.8828125" proc.memory.rssMB‚ 2338.6484375 cpu‚27.19594852379751 proc.cpu.threads‚17( disk./.usagePercent‚32.6944265091255 disk.sda3.out‚ 60.171875& disk.nvme0c0n1.out‚58530.431640625&P#(2ý- _wandbruntime‚3225  _runtime‚3225(µHÆþÀ trainer/global_step‚12099 % _timestamp‚1.7866618769289849e+09 _runtime‚3225.458998561 _step‚994 % lr-AdamW/pg1‚0.000525222370058711 % lr-AdamW/pg2‚0.000525222370058711þ½}ËÿÅ % lr-AdamW/pg1‚0.000525222370058711 % lr-AdamW/pg2‚0.000525222370058711 trainer/global_step‚12099 % _timestamp‚1.7866618769289849e+09 _runtime‚3225.458998561 _step‚994â‰Q2€- _wandbruntime‚3225  _runtime‚3225Ϋe!¿'¹O M@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚59049984 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚40618317 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚287865254084608 _runtime‚3225.463894253 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚41945088 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 2926216623616 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚44811699 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚61478912 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 @7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚5264 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚287865233440768 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚262 E8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚85430016 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 125829120 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚40622604 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17104896 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚322 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 2926216623616 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 3094386403328 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 122306526 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚322 $ _timestamp‚1.786661876929891e+09 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚333165217822208 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚41348020 PIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚75 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 4628521206 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚330238984093696 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚41945088 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚27899759 F>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 G>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4876 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 122905600 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚40622604 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚10155926 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 5494538240 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚4287 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 122905600 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 F?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚60 PCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚59049984 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 125829120 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚330239001198592 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 2926174678528 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚290959640487936 trainer/global_step‚12099 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚44811701 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚333165158772224 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4289 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚20643840 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚331205260206170 IBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚77 D;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚5264 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚322 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 KBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚4876 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚4287 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚40618317 BDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚333165158772224 IDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚27899759 F>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 G>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4876 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 122905600 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚40622604 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚10155926 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 5494538240 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚4287 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 122905600 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 F?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚60 PCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚59049984 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 125829120 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚330239001198592 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 2926174678528 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚290959640487936 trainer/global_step‚12099 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚44811701 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚333165158772224 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4289 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚20643840 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚331205260206170 IBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚77 D;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚5264 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚322 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 KBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚4876 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚4287 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚40618317 BDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚333165158772224 I5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5264 G@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚71 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 DDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚38058747 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 122306526 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚4295 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 4552982528 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 845178880 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚287888280813568 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚322 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚4293 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚85436856 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17104896 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚10156854 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 5494538240 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 122905600 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚322 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚262 G@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚60 K>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59053056 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 5368709120 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 125829120 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 3094631234560 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 D=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚60 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚40621604 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚429 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 2920355237506 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 5494538240 PDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521206 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚262 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚330265306431488 NADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚59053056 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚10156785 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 5368709120 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚44815254 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚330265306431488 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚287888301457408 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚61478912 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 % _timestamp‚1.7866618770922914e+09 _step‚996 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚40625897 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 2926405747200 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 2926447695360 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚44815254 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚27901964 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 125829120 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚85441151 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚223 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 2920313889438 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚262 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 2926447695360 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚41948160 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚40625897 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚40621604 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 NADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚58387428 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚322 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚41937920 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚331231734959746 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 5368709120 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚322 IBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚60 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 4654602240 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚333191771231744 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17104896 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚41348068 =7DeviceStatsMonitor.on_train_batch_end/segmentÖ]w}¦.all.freed‚0 BDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654602240^zŽAÏ&…ÉM K>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59053056 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 5368709120 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 125829120 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 3094631234560 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 D=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚60 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚40621604 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚429 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 2920355237506 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 5494538240 PDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521206 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚262 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚330265306431488 NADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚59053056 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚10156785 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 5368709120 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚44815254 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚330265306431488 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚287888301457408 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚61478912 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 % _timestamp‚1.7866618770922914e+09 _step‚996 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚40625897 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 2926405747200 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 2926447695360 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚44815254 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚27901964 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 125829120 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚85441151 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚223 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 2920313889438 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚262 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 2926447695360 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚41948160 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚40625897 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚40621604 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 NADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚58387428 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚322 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚41937920 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚331231734959746 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 5368709120 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚322 IBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚60 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 4654602240 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚333191771231744 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17104896 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚41348068 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 BDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654602240 E5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5264 G@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚71 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 DDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚38058747 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 122306526 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚4295 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 4552982528 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 845178880 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚287888280813568 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚322 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚4293 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚85436856 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17104896 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚10156854 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 5494538240 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 122905600 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚322 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚262 G@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚60äa¿1ƒ2†- _wandbruntime‚3225  _runtime‚3225ç) «t‡î 0loss/FRt1/assignment_loss‚0.6148477891642774 0loss/FRt2/detection_loss‚0.48771922896378606 trainer/global_step‚12099 % _timestamp‚1.7866618770934575e+09 _runtime‚3225.623253737 _step‚997 0loss/FRt2/assignment_loss‚0.6144249229978149 /loss/FRt1/detection_loss‚0.5344283677431323 %loss/total_loss‚2.251420308869011 epoch‚30ÕÄ:ºyˆó % _timestamp‚1.7866618770934575e+09 _runtime‚3225.623253737 _step‚997 0loss/FRt2/assignment_loss‚0.6144249229978149 /loss/FRt1/detection_loss‚0.5344283677431323 %loss/total_loss‚2.251420308869011 epoch‚30 0loss/FRt1/assignment_loss‚0.6148477891642774 0loss/FRt2/detection_loss‚0.48771922896378606 trainer/global_step‚12099åKÛv2‰- _wandbruntime‚3238  _runtime‚3238Àñ'jÇŠÁ _runtime‚ 3238.15764235 _step‚998 & lr-AdamW/pg1‚0.0005194952472393733 & lr-AdamW/pg2‚0.0005194952472393733 trainer/global_step‚12149 % _timestamp‚1.7866618896276164e+09ÒbçïÌ‹Æ & lr-AdamW/pg1‚0.0005194952472393733 & lr-AdamW/pg2‚0.0005194952472393733 trainer/global_step‚12149 % _timestamp‚1.7866618896276164e+09 _runtime‚ 3238.15764235 _step‚998æ×B6©2Œ- _wandbruntime‚3238  _runtime‚3238O #ëÀ'ºO RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 122905600 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚10197353 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚41945088 JBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚665 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 3106622033920 _step‚999 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚44989351 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚322 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 M@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚59049984 PCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚58387380 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚334492921477632 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 2937803389440 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚28010007 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚331555118088192 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17104896 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚334492862427648 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚4287 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 5368709120 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚331555100983296 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚223 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 5494538240 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚10197282 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚44989351 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚40787706 IDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4876 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚85777057 D;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚5264 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚38207362 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚20643840 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚61478912 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 4628521206 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚4289 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚4287 PCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚59049984 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚289017622724608 F>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4289 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚38207289 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 2937803389440 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 E8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚85772768 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚44989349 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚331555118088192 @7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚5264 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚40787706 PIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚71 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚332526111693578 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚322 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚334492862427648 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚41945088 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚40783419 BDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚332526111693578 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚322 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚334492862427648 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚41945088 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚40783419 BDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4876 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚85777057 D;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚5264 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚38207362 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚20643840 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚61478912 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 4628521206 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚4289 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚4287 PCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚59049984 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚289017622724608 F>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4289 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚38207289 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 2937803389440 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 E8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚85772768 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚44989349 `LDevice£éI]StatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚331555118088192ç£5¯ã2- _wandbruntime‚3238  _runtime‚3238‘rp'Ë&ÅM [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚331581440425984 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 NADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚58387428 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 EDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚38210305 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚28012214 NGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚82 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 122306526 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚322 D=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚60 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚334519474887168 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 122905600 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 831258624 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚61478912 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17104896 G@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚84 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚329620652359680 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 5494538240 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚331581440425984 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚40790999 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 5494538240 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚10198175 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚429 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 2937992513024 NADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚59053056 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 % _timestamp‚1.7866618897973628e+09 K>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59053056 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654602240 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚41948160 I@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚4876 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 DDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚85783903 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚40786706 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 3106864771072 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚38210389 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚289040670097408 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 3106915097600 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 IBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚60 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚223 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚50326528 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚40790999 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚331581423321088 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚85779608 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚289040649453568 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 B8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 5368709120 G@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚60 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚331581423321088 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚332552528059726 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 5368709120 >5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5264 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 2937992513024 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚20643840 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚262 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚262 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 2938034461184 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 4517085184 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚4293 B9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚5264 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚41948160 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚44992904 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚28012212 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚44992904 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 4654602240 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚334519474887168 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚334519415834112 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 4552982528 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 2938034461184 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚4293 SFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚70970368 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 4552982528 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 125829120 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚4295 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚44992902 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚209 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚322 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚332552586447154 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚329620669399040 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚322 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚85783903 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 5494538240 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚322 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚262 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚292147585195008 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521206 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 125829120 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 122905600 E8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 5368709120 G@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚60 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚331581423321088 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚332552528059726 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 5368709120 >5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5264 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 2937992513024 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚20643840 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚262 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚262 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 2938034461184 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 4517085184 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚4293 B9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚5264 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚41948160 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚44992904 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚28012212 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚44992904 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 4654602240 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚334519474887168 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚334519415834112 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 4552982528 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 2938034461184 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚4293 SFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚70970368 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 4552982528 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 125829120 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚4295 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚44992902 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚209 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚322 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚332552586447154 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚329620669399040 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚322 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚85783903 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 5494538240 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚322 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚262 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚292147585195008 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521206 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 125829120 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 122905600 EDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚38210305 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚28012214 NGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚82 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 122306526 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚322 D=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚60 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚334519474887168 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 122905600 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 831258624 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚61478912 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17104896 G@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚84 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚329620652359680 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 5494538240 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚331581440425984 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚40790999 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 5494538240 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚10198175 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚429 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 2937992513024 NADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚59053056 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 % _timestamp‚1.7866618897973628e+09 K>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59053056 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654602240 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚41948160 I@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚4876 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 DDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚85783903 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚40786706 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 3106864771072 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚38210389 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚289040670097408 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 3106915097600 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 IBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚60 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚223 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚50326528è\\2’- _wandbruntime‚3238  _runtime‚32384:Ít“î 0loss/FRt2/assignment_loss‚0.5381106077089979 /loss/FRt2/detection_loss‚0.5055619381194082 &loss/total_loss‚2.1445411649328334 % _timestamp‚1.7866618897983978e+09 _runtime‚ 3238.32853084 0loss/FRt1/assignment_loss‚0.6171674805624415 /loss/FRt1/detection_loss‚0.4837011385419859 epoch‚30 trainer/global_step‚12149 _step‚1001³Á8y”ó epoch‚30 trainer/global_step‚12149 _step‚1001 0loss/FRt2/assignment_loss‚0.5381106077089979 /loss/FRt2/detection_loss‚0.5055619381194082 &loss/total_loss‚2.1445411649328334 % _timestamp‚1.7866618897983978e+09 _runtime‚ 3238.32853084 0loss/FRt1/assignment_loss‚0.6171674805624415 /loss/FRt1/detection_loss‚0.4837011385419859éÛ½ –•‚(:‹ ƒ˜ùÓ‘Ö€á gpu.0.gpu‚61.0 gpu.0.memory‚11, gpu.0.memoryAllocated‚7.0313262939453125) gpu.0.memoryAllocatedBytes‚ 6039863296 gpu.0.temp‚35.0 gpu.0.powerWatts‚122.408' gpu.0.enforcedPowerLimitWatts‚500.0 gpu.0.powerPercent‚24.4816 gpu.0.smClock‚1410 gpu.0.memoryClock‚1593$ gpu.0.correctedMemoryErrors‚1955# gpu.0.uncorrectedMemoryErrors‚0Æ$Ï“–‚(:ˆ ƒ˜ùÓÓ¹óã# disk.nvme0c0n1.in‚ 43725.7578125 disk.sda.out‚ 60.3828125! disk.nvme0n1.in‚ 43725.7578125 disk.drbd2165.in‚0 disk.dm-0.in‚ 43725.7578125 proc.cpu.threads‚17 disk.sda1.out‚0 disk.sda2.out‚0% disk.nvme0n1.out‚58536.3916015625 disk.drbd2165.out‚0" disk.dm-0.out‚58536.3916015625$ memory_percent‚68.09039115905762" proc.memory.rssMB‚ 2338.6484375 cpu‚27.19541760034227 network.sent‚ 1.572203e+07 network.recv‚ 5.991632e+06) disk./.usagePercent‚32.69455348886248$ disk./.usageGB‚487.1724739074707 disk.sda3.out‚ 60.3828125' disk.nvme0c0n1.out‚58536.3916015625 disk.sda.in‚ 30344.4453125 disk.sda1.in‚0 disk.sda3.in‚ 30344.4453125 disk.sda2.in‚0 disk.dm-1.in‚ 30344.4453125 disk.dm-1.out‚ 60.3828125( proc.memory.availableMB‚ 5228.0703125* proc.memory.percent‚14.273977279663086Ðsðû2—- _wandbruntime‚3251  _runtime‚3251¾KéǘÁ & lr-AdamW/pg1‚0.0005137829293095726 & lr-AdamW/pg2‚0.0005137829293095726 trainer/global_step‚12199 $ _timestamp‚1.786661902576553e+09 _runtime‚ 3251.10656445 _step‚1002JhAÌ™Æ _runtime‚ 3251.10656445 _step‚1002 & lr-AdamW/pg1‚0.0005137829293095726 & lr-AdamW/pg2‚0.0005137829293095726 trainer/global_step‚12199 $ _timestamp‚1.786661902576553e+09êR‡¼…2š- _wandbruntime‚3251  _runtime‚3251õYºÄ5'›ºO XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚10238441 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚333847021568366 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚40952808 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚38358700 PIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚70 F?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚60 M@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚59049984 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 4552982528 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚45167001 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 4654602240 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚41945088 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚332871234977792 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4289 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚322 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 5368709120 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚332871234977792 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚332871217872896 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17104896 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 2943206938042 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 E8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚86115520 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚40952808 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚41945088 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚322 IBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚60 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚28120259 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚61478912 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚333846963180986 PCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚59049984 PCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚58387380 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚223 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚429 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚209 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 4628521206 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 125829120 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚45167001 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 122905600 IBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚72 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 845178880 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 5368709120 $ _timestamp‚1.786661902577454e+09 C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚4289 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17104896 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 2949348210176 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 125829120 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 2949390155264 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 2949348210176 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚335820566083072 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚290169970720768 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 5494538240 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 2949390155264 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚290169991364608 @7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚5264 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 4517085184 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 125829120 IDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚335820566083072 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚41348020 _runtime‚3251.110885021 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚86119809 IDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4876 JBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚665 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚38358628 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 2943248286062 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 5494538240 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚45166999 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚293288961009664 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 831258624 _step‚1003 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚262 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚40948521 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚28120257 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚20643840 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚335820625133056 KBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚4876 BDeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4289 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚322 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 5368709120 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚332871234977792 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚332871217872896 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17104896 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 2943206938042 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 E8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚86115520 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚40952808 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚41945088 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚322 IBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚60 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚28120259 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚61478912 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚333846963180986 PCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚59049984 PCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚58387380 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚223 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚429 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚209 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 4628521206 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 125829120 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚45167001 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 122905600 IBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚72 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 845178880 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 5368709120 $ _timestamp‚1.786661902577454e+09 C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚4289 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17104896 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 2949348210176 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 125829120 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 2949390155264 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 2949348210176 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚335820566083072 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚290169970720768 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 5494538240 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 2949390155264 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚290169991364608 @7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚5264 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 4517085184 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 125829120 IDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚335820566083072 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚41348020 _runtime‚3251.110885021 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚86119809 IDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4876 JBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚665 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚38358628 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 2943248286062 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 5494538240 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚45166999 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚293288961009664 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 831258624 _step‚1003 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚262 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚40948521 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚28120257 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚20643840 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚335820625133056 KBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚4876 B6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚322 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 EDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚86126655 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚429 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚293312254458880 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚290193038737408 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 NADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚58387428 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 122905600 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚293312183488512 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 3119215721472 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚335847178542592 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 4552982528 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚41948160 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 4552982528 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚41948160 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 3119165394944 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 125829120 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654602240 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 2949579278848 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚322 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚262 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17104896 G@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚80 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚10239244 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 4654602240 SFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚70970368 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚330929959075840 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚45170554 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 5494538240 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚45170552 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚20643840 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 5368709120 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚262 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚335847178542592 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚28122462 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 IBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚60 D5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5264 PDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚38361628 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 122306526 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 5494538240 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚45170554 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚40956101 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚332897540210688 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚322 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 831258624 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 4517085184 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚332897557315584 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚4293 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚332897540210688 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 2949579278848 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚50326528 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚333873379547134 % _timestamp‚1.7866619027410707e+09 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚290193018093568 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚322 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 5494538240 EDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59053056 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚28122464 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚10239166 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚330929942036480 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚4295 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 122905600 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 2949621227008 B9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚5264 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17104896 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚335847119489536 I@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚4876 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚38361708 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 5368709120 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 125829120 trainer/global_step‚12199 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚86126655 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚86122360 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚209 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521206 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 2943478858722 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 2943437510654 _runtime‚3251.271387408 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 125829120 G@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚602ÏÐ&ŸÊM KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17104896 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚335847119489536 I@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚4876 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚38361708 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 5368709120 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 125829120 trainer/global_step‚12199 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚86126655 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚86122360 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚209 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521206 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 2943478858722 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 2943437510654 _runtime‚3251.271387408 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 125829120 G@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚60 D=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚60 _step‚1004 NADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚59053056 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚41348068 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚322 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 EDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚86126655 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚429 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚293312254458880 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚290193038737408 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 NADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚58387428 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 122905600 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚293312183488512 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 3119215721472 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚335847178542592 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 4552982528 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚41948160 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 4552982528 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚41948160 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 3119165394944 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 125829120 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654602240 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 2949579278848 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚322 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚262 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17104896 G@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚80 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚10239244 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 4654602240 SFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚70970368 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚330929959075840 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚45170554 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 5494538240 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚45170552 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚20643840 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 5368709120 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚262 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚335847178542592 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚28122462 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 IBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚60 D5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5264 PDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚38361628 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 122306526 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 5494538240 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚45170554 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚40956101 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚332897540210688 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚322 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 831258624 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 4517085184 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚332897557315584 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚4293 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚332897540210688 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 2949579278848 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚50326528 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚333873379547134 % _timestamp‚1.7866619027410707e+09 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚290193018093568 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚322 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 5494538240 EDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59053056 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚28122464 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚10239166 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚330929942036480 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚4295 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 122905600 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 2949621227008 B9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚5264ìíÐ+“2 - _wandbruntime‚3251  _runtime‚3251fgëu¡ï /loss/FRt1/detection_loss‚0.5438628453090734 % _timestamp‚1.7866619027421467e+09 _step‚1005 0loss/FRt2/detection_loss‚0.48050609775558556 %loss/total_loss‚2.301053875777634 epoch‚30 trainer/global_step‚12199 _runtime‚3251.271944101 0loss/FRt1/assignment_loss‚0.6997974037751867 0loss/FRt2/assignment_loss‚0.5768875289377886 üýiz¢ô 0loss/FRt1/assignment_loss‚0.6997974037751867 0loss/FRt2/assignment_loss‚0.5768875289377886 /loss/FRt1/detection_loss‚0.5438628453090734 % _timestamp‚1.7866619027421467e+09 _step‚1005 0loss/FRt2/detection_loss‚0.48050609775558556 %loss/total_loss‚2.301053875777634 epoch‚30 trainer/global_step‚12199 _runtime‚3251.271944101íA‚wì–£‚(:‹ ’˜ùÓ¦Õ‘á gpu.0.gpu‚64.0 gpu.0.memory‚12, gpu.0.memoryAllocated‚7.0313262939453125) gpu.0.memoryAllocatedBytes‚ 6039863296 gpu.0.temp‚37.0 gpu.0.powerWatts‚219.051' gpu.0.enforcedPowerLimitWatts‚500.0 gpu.0.powerPercent‚43.8102 gpu.0.smClock‚1410 gpu.0.memoryClock‚1593$ gpu.0.correctedMemoryErrors‚1955# gpu.0.uncorrectedMemoryErrors‚0±$¤Ç›¤‚(: ’˜ùÓ¼‰öâ$ disk./.usageGB‚487.1723403930664 disk.drbd2165.in‚0 disk.dm-0.in‚43747.05859375! disk.dm-0.out‚58544.404296875$ disk.nvme0c0n1.in‚43747.05859375& disk.nvme0c0n1.out‚58544.404296875 disk.sda.in‚30458.71484375 network.recv‚ 6.013569e+06$ disk.nvme0n1.out‚58544.404296875 disk.dm-1.out‚ 60.64453125 disk.sda3.out‚ 60.64453125 disk.sda1.out‚0 disk.sda2.in‚0 disk.sda2.out‚0$ memory_percent‚68.09325218200684 disk.sda3.in‚30458.71484375" disk.nvme0n1.in‚43747.05859375 disk.dm-1.in‚30458.71484375 disk.drbd2165.out‚0 disk.sda1.in‚0* proc.memory.percent‚14.273977279663086 proc.cpu.threads‚17 network.sent‚ 1.5807725e+07 disk.sda.out‚ 60.64453125( proc.memory.availableMB‚ 5227.6015625" proc.memory.rssMB‚ 2338.6484375 cpu‚27.194923449029975) disk./.usagePercent‚32.69454452859878ñs82¥- _wandbruntime‚3263  _runtime‚3263T•ešÈ¦Â _step‚1006 & lr-AdamW/pg1‚0.0005080857831608144 & lr-AdamW/pg2‚0.0005080857831608144 trainer/global_step‚12249 $ _timestamp‚1.786661914801623e+09 _runtime‚3263.331646114²‘†<§Ç & lr-AdamW/pg1‚0.0005080857831608144 & lr-AdamW/pÍ$N¾‘g2‚0.0005080857831608144 trainer/global_step‚12249 $ _timestamp‚1.786661914801623e+09 _runtime‚3263.331646114 _step‚1006îs{?F2¨- _wandbruntime‚3263  _runtime‚3263v‘BÁ'©»O PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚41117910 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 4552982528 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚209 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 3131203161600 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 trainer/global_step‚12249 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 2960976921088 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 2960976921088 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚322 IBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚60 M@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚59049984 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17104896 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚332213062959104 KDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚60 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚4287 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 3131245102592 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚61478912 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚41113623 PIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚72 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚41940992 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 122306526 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚322 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚45344649 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚334187351867392 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚337148328788480 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚41945088 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚262 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17104896 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 4552982528 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚337148269738496 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚38509847 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚38509773 UHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚62584832 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚41348020 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 2954768748650 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 2960934976000 IDeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4289 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚294453542522368 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚262 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 2960934976000 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚45344651 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚41113623 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 4654602240 BDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚335167814668394 F?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚60 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 831258624 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚335167873055774 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 5368709120 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚262 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚337148269738496 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚334187351867392 IBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚74 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚294453605107200 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 % _timestamp‚1.7866619148026485e+09 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 125829120 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚41945088 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚45344649 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚10279266 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚4289 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚337148328788480 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 122905600 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚10279338 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 845178880 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚20643840 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚334187334762496 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚291322360004608 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 @7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚5264 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚334187334762496 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 122905600 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚429 G>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4876 PCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚59049984 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 5368709120 _step‚1007 PCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚58387380 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚332213045919744 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 5494538240 _runtime‚3263.332923703 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚45344651 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚86462561 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚41117910 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚223 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 5494538240 IDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚337148269738496 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚334187351867392 IBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚74 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚294453605107200 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 % _timestamp‚1.7866619148026485e+09 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 125829120 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚41945088 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚45344649 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚10279266 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚4289 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚337148328788480 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 122905600 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚10279338 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 845178880 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚20643840 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚334187334762496 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚291322360004608 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 @7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚5264 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚334187334762496 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 122905600 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚429 G>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4876 PCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚59049984 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 5368709120 _step‚1007 PCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚58387380 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚332213045919744 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 5494538240 _runtime‚3263.332923703 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚45344651 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚86462561 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚41117910 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚223 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 5494538240 IDeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4289 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚294453542522368 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚262 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 2960934976000 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚45344651 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚41113623 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 4654602240 BDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚335167814668394 F?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚60 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 831258624 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚335167873055774 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 5368709120 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚262ï ¦ 2«- _wandbruntime‚3263  _runtime‚3263„šÉ&¬ÃM RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 125829120 trainer/global_step‚12249 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚28232712 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 4654602240 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 122905600 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 3131496225280 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚4293 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 125829120 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚337174882198016 E8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚4295 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 122306526 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚337174882198016 DDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 NADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚59053056 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 5494538240 _step‚1008 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚41348068 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚334213657100288 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚334213674205184 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚38512750 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚294476903602688 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚335194231034542 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚332239248752640 D=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚60 PDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 5368709120 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚45348204 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 122905600 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚429 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚262 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 2961207992832 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 G@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚60 >5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5264 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17104896 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 5494538240 K>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59053056 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 4552982528 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚41121203 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚61478912 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521206 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚45348204 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 B6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚322 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚86469407 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚41116910 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚28232714 NGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚79 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚322 NADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚58387428 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚294476830535168 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚334213674205184 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚38512831 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 5494538240 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 5368709120 _runtime‚ 3263.4946254 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚337174823144960 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 2961166044672 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚10280117 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 % _timestamp‚1.7866619149643424e+09 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚45348202 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚45348202 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚41116910 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 4552982528 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17104896 SFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚73067520 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚322 EDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654602240 B9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚5264 H@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚665 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚223 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 5368709120 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 2961207992832 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚86469407 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚20643840 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚322 IBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚60¼=ÚŠ­ÈM YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 2961207992832 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚86469407 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚20643840 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚322 IBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚60 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 125829120 trainer/global_step‚12249 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚28232712 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 4654602240 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 122905600 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 3131496225280 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚4293 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 125829120 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚337174882198016 E8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚4295 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 122306526 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚337174882198016 DDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 NADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚59053056 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 5494538240 _step‚1008 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚41348068 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚334213657100288 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚334213674205184 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚38512750 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚294476903602688 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚335194231034542 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚332239248752640 D=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚60 PDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 5368709120 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚45348204 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 122905600 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚429 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚262 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 2961207992832 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 G@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚60 >5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5264 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17104896 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 5494538240 K>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59053056 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 4552982528 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚41121203 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚61478912 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521206 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚45348204 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 B6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚322 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚86469407 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚41116910 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚28232714 NGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚79 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚322 NADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚58387428 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚294476830535168 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚334213674205184 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚38512831 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 5494538240 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 5368709120 _runtime‚ 3263.4946254 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚337174823144960 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 2961166044672 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚10280117 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 % _timestamp‚1.7866619149643424e+09 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚45348202 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚45348202 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚41116910 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 4552982528 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17104896 SFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚73067520 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚322 EDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654602240 B9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚5264 H@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚665 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚223 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 5368709120𛕠Ó2®- _wandbruntime‚3263  _runtime‚3263.>µÒt¯î 0loss/FRt1/assignment_loss‚0.7036196328362505 0loss/FRt2/assignment_loss‚0.5778555549662829 /loss/FRt1/detection_loss‚0.4688490820783078 %loss/total_loss‚2.247825317576263 epoch‚30 $ _timestamp‚1.786661914965264e+09 _runtime‚3263.495057137 _step‚1009 0loss/FRt2/detection_loss‚0.49750104769542186 trainer/global_step‚12249½ž„y°ó _step‚1009 0loss/FRt2/detection_loss‚0.49750104769542186 trainer/global_step‚12249 0loss/FRt1/assignment_loss‚0.7036196328362505 0loss/FRt2/assignment_loss‚0.5778555549662829 /loss/FRt1/detection_loss‚0.4688490820783078 %loss/total_loss‚2.247825317576263 epoch‚30 $ _timestamp‚1.786661914965264e+09 _runtime‚3263.495057137ñ•°àa“±‚(:ˆ ¡˜ùÓ·¢á gpu.0.gpu‚63.0 gpu.0.memory‚12, gpu.0.memoryAllocated‚7.0313262939453125) gpu.0.memoryAllocatedBytes‚ 6039863296 gpu.0.temp‚37.0 gpu.0.powerWatts‚190.35' gpu.0.enforcedPowerLimitWatts‚500.0 gpu.0.powerPercent‚38.07 gpu.0.smClock‚1410 gpu.0.memoryClock‚1593$ gpu.0.correctedMemoryErrors‚1955# gpu.0.uncorrectedMemoryErrors‚0d-!÷¡²‚(:– ¡˜ùÓ•ÖÇã network.recv‚ 6.043342e+06 disk.dm-0.in‚43747.05859375 disk.sda.out‚ 60.7578125 disk.sda1.out‚0 disk.dm-1.in‚30458.71484375* proc.memory.percent‚14.273977279663086 cpu‚27.194633234827606 proc.cpu.threads‚17 network.sent‚ 1.5879893e+07* disk./.usagePercent‚32.694546320651526% disk./.usageGB‚487.17236709594727 disk.drbd2165.out‚0 disk.sda3.in‚30458.71484375$ disk.nvme0c0n1.in‚43747.05859375$ memory_percent‚68.09384822845459# disk.dm-0.out‚58550.42919921875 disk.sda3.out‚ 60.7578125 disk.sda.in‚30458.71484375 disk.sda1.in‚0 disk.sda2.in‚0& disk.nvme0n1.out‚58550.42919921875) proc.memory.availableMB‚ 5227.50390625" proc.memory.rssMB‚ 2338.6484375 disk.drbd2165.in‚0( disk.nvme0c0n1.out‚58550.42919921875 disk.sda2.out‚0" disk.nvme0n1.in‚43747.05859375 disk.dm-1.out‚ 60.7578125[7Þ2³- _wandbruntime‚3275  _runtime‚3275®xžÃÉ´Ã & lr-AdamW/pg1‚0.0005024041747101499 & lr-AdamW/pg2‚0.0005024041747101499 trainer/global_step‚12299 % _timestamp‚1.7866619271606524e+09 _runtime‚3275.690682281 _step‚1010›’ìÎµÈ & lr-AdamW/pg1‚0.0005024041747101499 & lr-AdamW/pg2‚0.0005024041747101499 trainer/global_step‚12299 % _timestamp‚1.7866619271606524e+09 _runtime‚3275.690682281 _step‚1010òÇ®Ô2¶- _wandbruntime‚3275  _runtime‚3275R—ÏTÀ'·ºO IDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚338475973393920 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 845178880 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 4654602240 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 4552982528 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚20643840 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 4517085184 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 5494538240 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚41278725 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚10320167 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚61478912 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚4287 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17104896 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 3143470551040 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 5368709120 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 2966371907278 PCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚59049984 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚335503451652096 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 F?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚60 trainer/global_step‚12299 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚335503468756992 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 3143518783488 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 4552982528 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚45522299 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚41283012 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚4287 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚28340759 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚333522352635904 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚262 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚335503451652096 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚338476032443904 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 2972563686912 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚292474728644608 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚336488666155802 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚262 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4289 UHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚68876288 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 PCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚58387380 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚322 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 KDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚60 _runtime‚3275.694867637 D;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚5264 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 122306526 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚262 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 2972521741824 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚223 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚45522301 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 2972563686912 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚429 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚295618247428096 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 125829120 $ _timestamp‚1.786661927161649e+09 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 122905600 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 4654602240 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚335503468756992 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 JBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚665 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚48232448 C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚4289 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 5494538240 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 5368709120 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 125829120 _step‚1011 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 M@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚59049984 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚209 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 831258624 @7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚5264 F>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 G>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4876 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚338475973393920 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 E8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚86801024 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚45522299 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚338476032443904 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 PIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚72 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 IBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚74 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚41348020 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 5494538240 IDeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4289 UHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚68876288 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 PCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚58387380 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚322 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 KDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚60 _runtime‚3275.694867637 D;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚5264 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 122306526 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚262 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 2972521741824 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚223 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚45522301 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 2972563686912 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚429 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚295618247428096 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 125829120 $ _timestamp‚1.786661927161649e+09 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 122905600 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 4654602240 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚335503468756992 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 JBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚665 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚48232448 C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚4289 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 5494538240 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 5368709120 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 125829120 _step‚1011 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 M@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚59049984 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚209 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 831258624 @7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚5264 F>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 G>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4876 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚338475973393920 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 E8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚86801024 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚45522299 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚338476032443904 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 PIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚72 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 IBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚74 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚41348020 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 5494538240 IDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚338475973393920 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 845178880 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 4654602240 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 4552982528 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚20643840 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 4517085184 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 5494538240 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚41278725 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚10320167 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚61478912 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚4287 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17104896 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 3143470551040 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 5368709120 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 2966371907278 PCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚59049984ó"Yº2¹- _wandbruntime‚3275  _runtime‚3275ÛÐ_FUºÅM UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 125829120 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 125829120 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 >5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5264 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚41282012 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚335529773989888 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚4293 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 4552982528 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 EDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521206 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 122905600 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 NADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚58387428 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚322 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654602240 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 122306526 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚262 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17104896 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 2966602479938 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚335529791094784 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚86812159 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚38663995 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 831258624 % _timestamp‚1.7866619273233867e+09 _step‚1012 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 4654602240 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚335529773989888 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚429 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚61478912 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 5368709120 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚41282012 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚28342962 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚10320962 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚333548521390080 IBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚60 D=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚60 trainer/global_step‚12299 I@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚4876 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 5494538240 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚338502585853440 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚45525852 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 845178880 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚292497755373568 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚45525852 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚333548538429440 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 2972794758656 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚41948160 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚223 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚86807864 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚336515082521950 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 5368709120 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚45525854 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 122905600 SFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚66776064 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17104896 EDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚38663924 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚336515140909378 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚41348068 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 5494538240 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚4293 K>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59053056 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚322 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚322 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚86812159 DDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚338502585853440 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚295641537403904 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚322 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚45525854 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 5368709120 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 5494538240 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 125829120 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 4552982528 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 4517085184 _runtime‚3275.853671642 PDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚38663924 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚336515140909378 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚41348068 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 5494538240 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚4293 K>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59053056 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚322 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚322 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚86812159 DDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚338502585853440 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚295641537403904 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚322 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚45525854 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 5368709120 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 5494538240 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 125829120 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 4552982528 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 4517085184 _runtime‚3275.853671642 P5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5264 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚41282012 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚335529773989888 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚4293 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 4552982528 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 EDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521206 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 122905600 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 NADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚58387428 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚322 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654602240 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 122306526 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚262 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17104896 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 2966602479938 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚335529791094784 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚86812159 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚38663995 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 831258624 % _timestamp‚1.7866619273233867e+09 _step‚1012 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 4654602240 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚335529773989888 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚429 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚61478912 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 5368709120 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚41282012 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚28342962 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚10320962 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚333548521390080 IBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚60 D=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚60 trainer/global_step‚12299 I@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚4876 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 5494538240 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚338502585853440 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚45525852 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 845178880 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚292497755373568 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚45525852 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚333548538429440 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 2972794758656 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚41948160 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚223 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚86807864 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚336515082521950 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 5368709120ô¾›òe2¼- _wandbruntime‚3275  _runtime‚3275@½iút½î 0loss/FRt1/assignment_loss‚0.5777356529435106 0loss/FRt2/assignment_loss‚0.5074550964795426 %loss/total_loss‚2.060354694111242 epoch‚30 $ _timestamp‚1.786661927324377e+09 _step‚1013 /loss/FRt1/detection_loss‚0.5019845995016718 0loss/FRt2/detection_loss‚0.47317934518651683 trainer/global_step‚12299 _runtime‚3275.854193329¬ºœy¾ó 0loss/FRt1/assignment_loss‚0.5777356529435106 0loss/FRt2/assignment_loss‚0.5074550964795426 %loss/total_loss‚2.060354694111242 epoch‚30 $ _timestamp‚1.786661927324377e+09 _step‚1013 /loss/FRt1/detection_loss‚0.5019845995016718 0loss/FRt2/detection_loss‚0.47317934518651683 trainer/global_step‚12299 _runtime‚3275.854193329õûJµò–¿‚(:‹ °˜ùӲˎá gpu.0.gpu‚63.0 gpu.0.memory‚12, gpu.0.memoryAllocated‚7.0313262939453125) gpu.0.memoryAllocatedBytes‚ 6039863296 gpu.0.temp‚37.0 gpu.0.powerWatts‚206.921' gpu.0.enforcedPowerLimitWatts‚500.0 gpu.0.powerPercent‚41.3842 gpu.0.smClock‚1410 gpu.0.memoryClock‚1593$ gpu.0.correctedMemoryErrors‚1955# gpu.0.uncorrectedMemoryErrors‚0¿l—M À‚(:• °˜ùÓðŒëâ disk.sda.in‚30458.71484375 disk.sda.out‚ 61.3203125) proc.memory.availableMB‚ 5165.58203125* proc.memory.percent‚14.274001121520996 cpu‚27.194519208227923 disk.drbd2165.in‚0$ disk./.usageGB‚487.1728706359863 disk.sda1.in‚0( disk.nvme0c0n1.out‚58558.82373046875$ memory_percent‚68.47178936004639 proc.cpu.threads‚17 network.recv‚ 6.067858e+06" disk.nvme0n1.in‚43748.32421875 disk.dm-0.in‚43748.32421875 disk.sda3.out‚ 61.3203125$ disk.nvme0c0n1.in‚43748.32421875# proc.memory.rssMB‚ 2338.65234375 disk.drbd2165.out‚0 network.sent‚ 1.5947699e+07 disk.sda1.out‚0 disk.sda2.in‚0 disk.sda2.out‚0& disk.nvme0n1.out‚58558.82373046875 disk.dm-1.out‚ 61.3203125# disk.dm-0.out‚58558.82373046875) disk./.usagePercent‚32.69458011364604 disk.dm-1.in‚30458.71484375 disk.sda3.in‚30458.71484375<Ÿe2Á- _wandbruntime‚3288  _runtime‚3288´8#È & lr-AdamW/pg1‚0.0004967384688766737 & lr-AdamW/pg2‚0.0004967384688766737 trainer/global_step‚12349 $ _timestamp‚1.786661939489798e+09 _runtime‚3288.019888067 _step‚1014÷UåÉÍÃÇ _runtime‚3288.019888067 _step‚1014 & lr-AdamW/pg1‚0.0004967384688766737 & lr-AdamW/pg2‚0.0004967384688766737 trainer/global_step‚12349 $ _timestamp‚1.786661939489798e+09ö ¥º2Ä- _wandbruntime‚3288  _runtime‚3288Ðá—xÀ'źO UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 5368709120 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚339803677049344 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚41945088 JBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚665 IBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚65 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 5368709120 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚296782861896704 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚293627097284608 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 125829120 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚45699949 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚4287 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚322 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚322 _runtime‚3288.024916947 M@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚59049984 PCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚59049984 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 UHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚62584832 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 125829120 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚41945088 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚87148065 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚293627076640768 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 125829120 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 IBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚60 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚45699951 G>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4876 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 2977933717886 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚28451007 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17104896 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 4654602240 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 2984150452736 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4289 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚209 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚322 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 4552982528 PIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚63 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 4517085184 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚45699951 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚41940992 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚61478912 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚4289 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚41443827 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚262 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 122306526 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 5494538240 E8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚87143776 @7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚5264 F>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 831258624 _step‚1015 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 IŠQÆ¿O TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚41945088 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚87148065 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚293627076640768 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 125829120 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 IBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚60 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚45699951 G>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4876 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 2977933717886 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚28451007 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17104896 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 4654602240 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 2984150452736 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4289 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚209 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚322 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 4552982528 PIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚63 VGDevicmOÜt eStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 4517085184 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚45699951 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚41940992 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚61478912 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚4289 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚41443827 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚262 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 122306526 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 5494538240 E8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚87143776 @7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚5264 F>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 831258624 _step‚1015 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 IDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚339803677049344 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚41945088 JBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚665 IBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚65 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 5368709120 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚296782861896704 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚293627097284608 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 125829120 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚45699949 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚4287 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚322 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚322 _runtime‚3288.024916947 M@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚59049984 PCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚59049984 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 UHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚62584832 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 125829120÷Ôq<ð2Ç- _wandbruntime‚3288  _runtime‚3288¢8rÊ&ÈÄM NADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚59053056 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 5368709120 D=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚60 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚45703502 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚336845907984384 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 831258624 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚322 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚41948160 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17104896 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚296806090748928 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚87150616 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚293650124013568 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚262 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚28453212 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚339830230455808 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚10362137 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 >5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5264 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚41948160 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 2984339576320 H@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚665 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚41451407 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚296806153330688 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚41937920 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 I@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚4876 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚4293 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚28453214 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚61478912 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 _runtime‚3288.188023274 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 122905600 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚10362199 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 2978122942478 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚334857828106240 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 5368709120 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚322 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚336845890879488 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚209 SFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚62581760 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚337835992396786 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚41451407 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚336845890879488 B9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚5264 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚87154911 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚87154911 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚45703504 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚4293 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 125829120 G@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚60 PDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59053056 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17104896 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 2984381524480 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚262 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚41447114 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚429 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 4552982528 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚45703502 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚38815349 NADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚58387428 trainer/global_step‚12349 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654602240 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚336845907984384 G@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚64 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚41348068 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 4654602240 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 2984381524480 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 122905600 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 4517085184 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 5368709120 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚41447114 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 845178880 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚322 D6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚322 IBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚60 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚38815413 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 3155966735360 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚337835934009358 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚4295 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚45703504 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚262 NGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚62 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521206 _step‚1016:ÓŠÏ&ÉÉM cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚293650144657408 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 125829120 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚339830289508864 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 3156008673280 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚322 IBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚60 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚38815413 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 3155966735360 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚337835934009358 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚4295 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚45703504 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚262 NGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚62 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521206 _step‚1016 NADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚59053056 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 5368709120 D=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚60 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚45703502 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚336845907984384 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 831258624 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚322 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚41948160 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17104896 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚296806090748928 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚87150616 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚293650124013568 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚262 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚28453212 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚339830230455808 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚10362137 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 >5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5264 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚41948160 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 2984339576320 H@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚665 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚41451407 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚296806153330688 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚41937920 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 I@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚4876 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚4293 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚28453214 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚61478912 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 _runtime‚3288.188023274 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 122905600 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚10362199 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 2978122942478 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚334857828106240 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 5368709120 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚322 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚336845890879488 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚209 SFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚62581760 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚337835992396786 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚41451407 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚336845890879488 B9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚5264 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚87154911 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚87154911 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚45703504 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚4293 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 125829120 G@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚60 PDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59053056 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17104896 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 2984381524480 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚262 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚41447114 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚429 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 4552982528 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚45703502 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚38815349 NADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚58387428 trainer/global_step‚12349 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654602240 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚336845907984384 G@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚64 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚41348068 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 4654602240 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 2984381524480 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 122905600 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 4517085184 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 5368709120 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚41447114 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 845178880 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚322 DDeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4289 D;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚5264 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 125829120 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚341131380704768 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚41945088 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚4287 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 122905600 PCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚58387380 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚38963756 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚38963677 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚209 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 trainer/global_step‚12399 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 4552982528 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚223 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚294779465924608 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚336140914950144 E8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚87486528 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚45877601 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚45877599 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚20643840 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 3168059050496 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚322 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚262 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚87490817 UHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚68876288 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚48232448 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚297947524975104 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚339130427517998 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 122306526 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 125829120 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚28561257 M@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚59049984 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17104896 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚341131380704768 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 4552982528 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 2995695273472 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚28561259 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 2995737218560 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17104896 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 2995737218560 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 3168010818048 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚322 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 5494538240 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 F>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚45877599 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 4628521206 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 5494538240 _runtime‚3300.282861922 IBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚79 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚41348020 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚45877601 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚41613216 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 5368709120 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚322 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚338135685431296 IDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚336140931989504 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 2989454180474 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 2995695273472 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚41613216 G>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4876 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚41945088îô”äÅ'Ô¿O TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚41945088 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚4287 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 122905600 PCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚58387380 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚38963756 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚38963677 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚209 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 trainer/global_step‚12399 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 4552982528 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚223 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚294779465924608 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚336140914950144 E8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚87486528 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚45877601 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚45877599 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚20643840 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 3168059050496 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚322 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚262 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚87490817 UHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚68876288 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚48232448 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚297947524975104 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚339130427517998 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 122306526 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 125829120 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚28561257 M@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚59049984 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17104896 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚341131380704768 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 4552982528 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 2995695273472 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚28561259 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 2995737218560 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17104896 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 2995737218560 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 3168010818048 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚322 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 5494538240 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 F>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚45877599 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 4628521206 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 5494538240 _runtime‚3300.282861922 IBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚79 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚41348020 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚45877601 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚41613216 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 5368709120 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚322 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚338135685431296 IDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚336140931989504 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 2989454180474 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 2995695273472 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚41613216 G>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4876 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚41945088 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚294779445280768 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 4517085184 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚262 _step‚1019 C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚4289 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚10402497 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 5494538240 $ _timestamp‚1.786661951749724e+09 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 4654602240 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚61478912 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚338135685431296 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 122905600 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚429 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 2989495528494 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 5368709120 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚338135702536192 JBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚665 PCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚59049984 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 4654602240 KBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚4876 PIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚77 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 831258624 BDeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4289 D;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚5264 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 125829120 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚341131380704768ûÇ&ƒÅ2Õ- _wandbruntime‚3300  _runtime‚3300æ^fË&ÖÅM LDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚223 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 2995926342144 I@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚4876 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚294802513297408 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚61478912 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚4293 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚339156843884194 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 5494538240 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚429 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚45881152 NADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚59053056 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 2995926342144 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚45881154 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚41616509 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚339156785496766 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚262 IBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚60 B9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚5264 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521206 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 2989684753086 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 5368709120 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚4295 PDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚87497663 G@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚72 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚41612216 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 4654602240 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚45881152 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚38966737 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚209 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚20643840 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 3168298147328 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚41948160 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚45881154 H@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚665 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚262 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚338162024873984 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 4552982528 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 122306526 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚28563464 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚87497663 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 4552982528 % _timestamp‚1.7866619519124982e+09 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚87493368 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚41948160 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚297970748862976 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 5368709120 K>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59053056 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚322 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 845178880 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 G@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚60 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚338162024873984 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚41937920 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 5368709120 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 125829120 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 3168256209408 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 BDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 122905600 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654602240 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚87493368 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17104896 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 122905600 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚294802492653568 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 4517085184 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 5494538240 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚322 _runtime‚3300.442864883 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚322 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚262 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 125829120 >5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5264 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 2995968290304<[È¢r×ÊM A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚4295 PDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚87497663 G@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚72 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚41612216 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 4654602240 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚45881152 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚38966737 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚209 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚20643840 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 3168298147328 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚41948160 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚45881154 H@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚665 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚262 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚338162024873984 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 4552982528 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 122306526 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚28563464 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚87497663 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 4552982528 % _timestamp‚1.7866619519124982e+09 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚87493368 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚41948160 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚297970748862976 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 5368709120 K>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59053056 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚322 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 845178880 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 G@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚60 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚338162024873984 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚41937920 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 5368709120 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 125829120 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 3168256209408 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 BDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 122905600 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654602240 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚87493368 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17104896 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 122905600 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚294802492653568 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 4517085184 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 5494538240 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚322 _runtime‚3300.442864883 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚322 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚262 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 125829120 >5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5264 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 2995968290304 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚223 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 2995926342144 I@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚4876 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚294802513297408 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚61478912 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚4293 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚339156843884194 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 5494538240 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚429 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚45881152 NADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚59053056 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 2995926342144 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚45881154 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚41616509 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚339156785496766 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚262 IBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚60 B9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚5264 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521206 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 2989684753086 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 5368709120üEÒÏ»2Ø- _wandbruntime‚3300  _runtime‚3300c|«‚uÙï /loss/FRt1/detection_loss‚0.5101121388570103 epoch‚30 trainer/global_step‚12399 % _timestamp‚1.7866619519135127e+09 0loss/FRt1/assignment_loss‚0.6648870893285503 0loss/FRt2/assignment_loss‚0.6210921830471392 /loss/FRt2/detection_loss‚0.4984097849875666 &loss/total_loss‚2.2945011962202666 _runtime‚3300.443359989 _step‚1021‘¬+zÚô /loss/FRt1/detection_loss‚0.5101121388570103 epoch‚30 trainer/global_step‚12399 % _timestamp‚1.7866619519135127e+09 0loss/FRt1/assignment_loss‚0.6648870893285503 0loss/FRt2/assignment_loss‚0.6210921830471392 /loss/FRt2/detection_loss‚0.4984097849875666 &loss/total_loss‚2.2945011962202666 _runtime‚3300.443359989 _step‚1021ý¡¶ª2Û- _wandbruntime‚3305  _runtime‚3305¤ymXÛÜÕ .Purity/*FRt/FRt_purity‚0.49061256647109985 )jet/accuracy_2_of_2‚0.226008802652359 particle/accuracy_1_of_1‚1 /particle/accuracy_2_of_2‚0.7090442180633545 'particle/f_score‚0.5509770512580872 % _timestamp‚1.7866619570066326e+09 _step‚1022 /Purity/*FRt/event_purity‚0.4766389727592468 -Purity/1FRt/FRt_purity‚0.4781198501586914 +particle/sensitivity‚0.6371459364891052 3Purity/0FRt/event_proportion‚0.3301943838596344 (particle/accuracy‚0.6314290165901184 )validation_accuracy‚0.226008802652359 /Purity/1FRt/event_purity‚0.4781198501586914 +particle/specificity‚0.6282724738121033 2Purity/*FRt/event_proportion‚0.669805645942688 /particle/accuracy_1_of_2‚0.9246811866760254 epoch‚30 _runtime‚3305.537095889 )loss/val_total_loss‚2.382768053857417 /Purity/2FRt/event_purity‚0.4524987041950226 2Purity/2FRt/event_proportion‚0.040062315762043 Purity/0FRt/event_purity‚1 3Purity/1FRt/event_proportion‚0.6297432780265808 *jet/accuracy_1_of_1‚0.3169178366661072 trainer/global_step‚12399 -Purity/2FRt/FRt_purity‚0.5881255865097046 *jet/accuracy_1_of_2‚0.6026231050491333 7validation_average_jet_accuracy‚0.32274606823921204Ô˜tàÝÚ % _timestamp‚1.7866619570066326e+09 _step‚1022 /Purity/*FRt/event_purity‚0.4766389727592468 -Purity/1FRt/FRt_purity‚0.4781198501586914 +particle/sensitivity‚0.6371459364891052 3Purity/0FRt/event_proportion‚0.3301943838596344 (particle/accuracy‚0.6314290165901184 )validation_accuracy‚0.226008802652359 /Purity/1FRt/event_purity‚0.4781198501586914 +particle/specificity‚0.6282724738121033 2Purity/*FRt/event_proportion‚0.669805645942688 /particle/accuracy_1_of_2‚0.9246811866760254 epoch‚30 _runtime‚3305.537095889 )loss/val_total_loss‚2.382768053857417 /Purity/2FRt/event_purity‚0.4524987041950226 2Purity/2FRt/event_proportion‚0.040062315762043 Purity/0FRt/event_purity‚1 3Purity/1FRt/event_proportion‚0.6297432780265808 *jet/accuracy_1_of_1‚0.3169178366661072 trainer/global_step‚12399 -Purity/2FRt/FRt_purity‚0.5881255865097046 *jet/accuracy_1_of_2‚0.6026231050491333 7validation_average_jet_accuracy‚0.32274606823921204 .Purity/*FRt/FRt_purity‚0.49061256647109985 )jet/accuracy_2_of_2‚0.226008802652359 particle/accuracy_1_of_1‚1 /particle/accuracy_2_of_2‚0.7090442180633545 'particle/f_score‚0.5509770512580872þ«ÈÐ| Þ‚(:• ΘùÓÂ¥üà gpu.0.gpu‚40.0 gpu.0.memory‚6, gpu.0.memoryAllocated‚7.0313262939453125) gpu.0.memoryAllocatedBytes‚ 6039863296 gpu.0.temp‚36.0 gpu.0.powerWatts‚127.229' gpu.0.enforcedPowerLimitWatts‚500.0) gpu.0.powerPercent‚25.445800000000002 gpu.0.smClock‚1410 gpu.0.memoryClock‚1593$ gpu.0.correctedMemoryErrors‚1955# gpu.0.uncorrectedMemoryErrors‚0ž…Ë’¢ß‚(:— ΘùÓ³…Þã% disk./.usageGB‚487.17935943603516 disk.sda2.in‚0& disk.nvme0n1.out‚58576.61572265625$ disk.nvme0c0n1.in‚43748.32421875$ memory_percent‚68.47243309020996# proc.memory.rssMB‚ 2338.65234375 cpu‚27.20163848545503 network.recv‚ 6.126878e+06 disk.dm-1.out‚ 61.65234375 disk.drbd2165.out‚0# disk.dm-0.out‚58576.61572265625 disk.sda3.in‚30458.71484375 disk.sda3.out‚ 61.65234375( disk.nvme0c0n1.out‚58576.61572265625 disk.sda1.in‚0 network.sent‚ 1.6091928e+07 disk.dm-1.in‚30458.71484375 disk.drbd2165.in‚0 disk.sda.in‚30458.71484375 disk.sda.out‚ 61.65234375 disk.sda1.out‚0( proc.memory.availableMB‚ 5165.4765625* proc.memory.percent‚14.274001121520996) disk./.usagePercent‚32.69501558246179 disk.sda2.out‚0" disk.nvme0n1.in‚43748.32421875 disk.dm-0.in‚43748.32421875 proc.cpu.threads‚17r\x‘2à- _wandbruntime‚3317  _runtime‚3317,wqÉáà & lr-AdamW/pg2‚0.0004854562196073105 trainer/global_step‚12449 % _timestamp‚1.7866619694579628e+09 _runtime‚3317.988097018 _step‚1023 & lr-AdamW/pg1‚0.0004854562196073105 &%,ÎâÈ trainer/global_step‚12449 % _timestamp‚1.7866619694579628e+09 _runtime‚3317.988097018 _step‚1023 & lr-AdamW/pg1‚0.0004854562196073105 & lr-AdamW/pg2‚0.0004854562196073105ÿ+áÛ2ã- _wandbruntime‚3317  _runtime‚3317~‘~À'äºO ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 3005123311203 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚10452859 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚41796218 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚10452789 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚209 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚322 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 3005081963183 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 5368709120 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 125829120 E8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚87912073 G>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4876 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 831258624 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚39164910 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚46115857 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 4654602240 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 3011391708160 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚322 C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚4289 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 3011349763072 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 5494538240 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 5368709120 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 125829120 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚41800505 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚20643840 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚339842756640768 D;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚5264 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚337831121158144 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚429 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 3184433191424 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 5494538240 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚342854148348928 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚342854089298944 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4289 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚322 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚39164982 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚41796218 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚342854148348928 PCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚59049984 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚339842739535872 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 5494538240 @7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚5264 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 3184380764672 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚61478912 BDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 4552982528 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 3011349763072 JBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚665 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚340836244469347 trainer/global_step‚12449 IDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 4517085184 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚299415464993280 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚262 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚41800505 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚299415538063872 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚337831104118784 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 122306526 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚87916362 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚322 KDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚60 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 125829120 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚41945088 KBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚4876 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚28712123 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 845178880 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚340836186081967 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 5368709120 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚339842739535872 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 122905600 IDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚342854089298944 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17104896 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚41348020 F?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚60 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚339842756640768 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚41945088 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚296231084228608 _runtime‚ 3317.99289057 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 IBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚60‹ijÅ'å¿O YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚342854148348928 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚342854089298944 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4289 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚322 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚39164982 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚41796218 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚342854148348928 PCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚59049984 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚339842739535872 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 5494538240 @7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚5264 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 3184380764672 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚61478912 BDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 4552982528 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 3011349763072 JBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚665 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚340836244469347 trainer/global_step‚12449 IDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 4517085184 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚299415464993280 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚262 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚41800505 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚299415538063872 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚337831104118784 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 122306526 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚87916362 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚322 KDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚60 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 125829120 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚41945088 KBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚4876 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚28712123 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 845178880 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚340836186081967 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 5368709120 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚339842739535872 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 122905600 IDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚342854089298944 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17104896 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚41348020 F?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚60 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚339842756640768 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚41945088 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚296231084228608 _runtime‚ 3317.99289057 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 IBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚60 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 3005123311203 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚10452859 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚41796218 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚10452789 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚209 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚322 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 3005081963183 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 5368709120 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 125829120 E8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚87912073 G>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4876 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 831258624 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚39164910 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚46115857 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 4654602240 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 3011391708160 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚322 C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚4289 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 3011349763072 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 5494538240 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 5368709120 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 125829120 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚41800505 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚20643840 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚339842756640768 D;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚5264 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚337831121158144 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚429 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 3184433191424 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 5494538240€*ßjè2æ- _wandbruntime‚3318  _runtime‚3318[8)çÄM ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚339869078978560 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 122905600 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 122306526 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚46119410 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚87923208 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚41803798 EDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚223 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚296254152245248 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚48229376 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 3005312535795 PDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521206 DDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59053056 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 845178880 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚322 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 _runtime‚ 3318.15739983 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚342880701758464 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚4295 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 4552982528 _step‚1025 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚41799505 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚342880701758464 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 122905600 H@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚665 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚41799505 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 5368709120 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚262 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17104896 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 3184675958272 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 5368709120 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚262 D=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚60 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚39167941 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚296254131601408 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 831258624 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚322 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚209 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚46119408 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 5368709120 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚87923208 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚20643840 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 3011580831744 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚39168003 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚61478912 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚337857289912320 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚4293 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚41948160 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 3184627728896 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 125829120 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚10453675 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚46119410 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚299438828203520 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 5494538240 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 5494538240 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚322 G@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚60 B9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚5264 SFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚68873216 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚340862602448115 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚41948160 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 3011580831744 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚41803798 NGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚60 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 3005353883863 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 5494538240 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚322 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17104896 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚28714328 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚299438759330304 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚262 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654602240 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚28714326 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚10453615 B5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5264 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚46119408 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 125829120 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚87918913 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚339869061873664 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 4654602240 I@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚4876 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚429 NADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚58387428 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚41348068 trainer/global_step‚12449 ,¥Ï&èÉM NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚46119410 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚299438828203520 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 5494538240 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 5494538240 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚322 G@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚60 B9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚5264 SFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚68873216 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚340862602448115 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚41948160 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 3011580831744 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚41803798 NGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚60 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 3005353883863 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 5494538240 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚322 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17104896 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚28714328 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚299438759330304 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚262 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654602240 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚28714326 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚10453615 B5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5264 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚46119408 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 125829120 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚87918913 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚339869061873664 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 4654602240 I@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚4876 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚429 NADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚58387428 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚41348068 trainer/global_step‚12449 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚339869078978560 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 122905600 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 122306526 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚46119410 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚87923208 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚41803798 EDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚223 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚296254152245248 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚48229376 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 3005312535795 PDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521206 DDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59053056 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 845178880 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚322 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 _runtime‚ 3318.15739983 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚342880701758464 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚4295 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 4552982528 _step‚1025 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚41799505 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚342880701758464 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 122905600 H@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚665 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚41799505 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 5368709120 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚262 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17104896 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 3184675958272 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 5368709120 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚262 D=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚60 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚39167941 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚296254131601408 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 831258624 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚322 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚209 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚46119408 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 5368709120 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚87923208 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚20643840 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 3011580831744 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚39168003 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚61478912 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚337857289912320 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚4293 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚41948160 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 3184627728896 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 125829120 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚10453675ÏsçS2é- _wandbruntime‚3318  _runtime‚3318º9Þtêî 0loss/FRt2/assignment_loss‚0.5173243407253406 /loss/FRt2/detection_loss‚0.4772096956279965 trainer/global_step‚12449 _runtime‚3318.158203359 _step‚1026 0loss/FRt1/assignment_loss‚0.6436921035803886 /loss/FRt1/detection_loss‚0.5355919940530093 %loss/total_loss‚2.173818133986735 epoch‚31 % _timestamp‚1.7866619696282005e+09Si*úyëó /loss/FRt1/detection_loss‚0.5355919940530093 %loss/total_loss‚2.173818133986735 epoch‚31 % _timestamp‚1.7866619696282005e+09 0loss/FRt2/assignment_loss‚0.5173243407253406 /loss/FRt2/detection_loss‚0.4772096956279965 trainer/global_step‚12449 _runtime‚3318.158203359 _step‚1026 0loss/FRt1/assignment_loss‚0.6436921035803886‚ö*Ÿ(–ì‚(:‹ ݘùÓË”„á gpu.0.gpu‚64.0 gpu.0.memory‚12, gpu.0.memoryAllocated‚7.0313262939453125) gpu.0.memoryAllocatedBytes‚ 6039863296 gpu.0.temp‚36.0 gpu.0.powerWatts‚171.328' gpu.0.enforcedPowerLimitWatts‚500.0 gpu.0.powerPercent‚34.2656 gpu.0.smClock‚1410 gpu.0.memoryClock‚1593$ gpu.0.correctedMemoryErrors‚1955# gpu.0.uncorrectedMemoryErrors‚0ññ–í‚(:‹ ݘùÓ·Š¦ä$ disk.nvme0c0n1.in‚43748.32421875 disk.sda.out‚ 61.76953125 disk.sda1.in‚0 disk.sda2.out‚0" disk.nvme0n1.in‚43748.32421875" disk.nvme0n1.out‚ 58581.7890625( proc.memory.availableMB‚ 5165.7109375* proc.memory.percent‚14.274001121520996) disk./.usagePercent‚32.69502070261247 disk.sda3.in‚30458.71484375 disk.sda3.out‚ 61.76953125$ disk.nvme0c0n1.out‚ 58581.7890625 disk.sda1.out‚0 disk.sda2.in‚0 disk.dm-1.in‚30458.71484375 disk.drbd2165.out‚0 network.sent‚ 1.6159669e+07 network.recv‚ 6.150485e+06 disk.sda.in‚30458.71484375 disk.drbd2165.in‚0 disk.dm-0.out‚ 58581.7890625$ memory_percent‚68.47100257873535# proc.memory.rssMB‚ 2338.65234375 proc.cpu.threads‚17% disk./.usageGB‚487.17943572998047 disk.dm-1.out‚ 61.76953125 disk.dm-0.in‚43748.32421875 cpu‚27.20110206083087K 2î- _wandbruntime‚3330  _runtime‚3330Ëh ±ËïÅ ' lr-AdamW/pg2‚0.00047984040080921154 trainer/global_step‚12499 % _timestamp‚1.7866619821872272e+09 _runtime‚3330.717259293 _step‚1027 ' lr-AdamW/pg1‚0.00047984040080921154`ÇP)ÐðÊ % _timestamp‚1.7866619821872272e+09 _runtime‚3330.717259293 _step‚1027 ' lr-AdamW/pg1‚0.00047984040080921154 ' lr-AdamW/pg2‚0.00047984040080921154 trainer/global_step‚12499ƒÓÿM/2ñ- _wandbruntime‚3330  _runtime‚3330K^Á'ò»O C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚4289 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚39316182 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 3196707511808 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚262 trainer/global_step‚12499 D;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚5264 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚20643840 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 5494538240 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 5368709120 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 125829120 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚88259114 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚54523904 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 3196652987904 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚322 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4289 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚300580181024256 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 4654602240 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚28822371 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 4552982528 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚429 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚339140410834944 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 % _timestamp‚1.7866619821882231e+09 KBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚4876 BDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚344181792954368 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 3022978473984 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚262 KDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚60 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚41965607 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚46293507 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 4654602240 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚61478912 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚41965607 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 IBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚60 _step‚1028 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚41945088 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 122905600 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚4287 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚322 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚297383452868608 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 5494538240 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 125829120 IDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 3016685121811 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 5494538240 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚344181852004352 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚10493725 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚297383473512448 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 122306526 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚344181852004352 @7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚5264 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚41961320 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 4552982528 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 125829120 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚262 G>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4876 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 3022936528896 IDeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4289 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚300580181024256 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 4654602240 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚28822371 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 SDDeviceSta8·?œ"tsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 4552982528 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚429 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚339140410834944 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 % _timestamp‚1.7866619821882231e+09 KBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚4876 BDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚344181792954368 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 3022978473984 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚262 KDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚60 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚41965607 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚46293507 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 4654602240 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚61478912 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚41965607 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 IBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚60 _step‚1028 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚41945088 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 122905600 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚4287 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚322 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚297383452868608 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 5494538240 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 125829120 IDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 3016685121811 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 5494538240 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚344181852004352 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚10493725 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚297383473512448 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 122306526 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚344181852004352 @7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚5264 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚41961320 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 4552982528 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 125829120 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚262 G>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4876 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 3022936528896 I6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚322 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚262 EDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521206 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 5494538240 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 IBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚60 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654602240 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 3023167597568 H@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚665 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 trainer/global_step‚12499 _step‚1029 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 4552982528 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 3016874346403 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚46297058 _runtime‚3330.886009074 >5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5264 EDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚39319102 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚20643840 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 4517085184 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚4293 K>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59053056 PDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 DDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 D=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚60 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚341185195868160 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 3016915694471 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 831258624 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 5494538240 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 BDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 5368709120 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚322 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 NADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚58387428 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 5494538240 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 % _timestamp‚1.7866619823555422e+09 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚4295 NGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚69 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚10494526 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 125829120 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 3023167597568 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚39319173 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚88265960 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚41964607 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚342183512322951 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚342183453935523 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 125829120 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚46297060 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚88261665 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚429 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚88265960 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚344208405413888 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚300603470803456 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚41964607 B9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚5264 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚300603404027392 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 3196949918208 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 125829120 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚341185195868160 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚341185178763264 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚41968900 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚339166579589120 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚41348068 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚88261665 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17104896 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚10494595 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 845178880 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚262 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚28824576q–ÝÐ&öÊM ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚341185195868160 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚341185178763264 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚41968900 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚339166579589120 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚41348068 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚88261665 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17104896 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚10494595 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 845178880 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚262 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚28824576 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 4654602240 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17104896 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚41948160 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 122905600 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚46132224 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 3023209545728 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚322 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚262 EDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521206 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 5494538240 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 IBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚60 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654602240 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 3023167597568 H@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚665 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 trainer/global_step‚12499 _step‚1029 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 4552982528 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 3016874346403 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚46297058 _runtime‚3330.886009074 >5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5264 EDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚39319102 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚20643840 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 4517085184 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚4293 K>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59053056 PDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 DDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 D=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚60 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚341185195868160 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 3016915694471 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 831258624 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 5494538240 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 BDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 5368709120 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚322 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 NADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚58387428 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 5494538240 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 % _timestamp‚1.7866619823555422e+09 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚4295 NGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚69 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚10494526 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 125829120 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 3023167597568 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚39319173 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚88265960 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚41964607 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚342183512322951 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚342183453935523 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 125829120 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚46297060 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚88261665 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚429 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚88265960 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚344208405413888 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚300603470803456 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚41964607 B9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚5264 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚300603404027392 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 3196949918208 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 125829120…;º2÷- _wandbruntime‚3330  _runtime‚3330täëôtøî /loss/FRt1/assignment_loss‚0.566121008233537 0loss/FRt2/assignment_loss‚0.5081988242371221 .loss/FRt1/detection_loss‚0.514293058232753 0loss/FRt2/detection_loss‚0.49169272657055074 &loss/total_loss‚2.0803056172739627 epoch‚31 trainer/global_step‚12499 _runtime‚3330.886545692 % _timestamp‚1.7866619823567312e+09 _step‚1030 %Èyùó epoch‚31 trainer/global_step‚12499 _runtime‚3330.886545692 % _timestamp‚1.7866619823567312e+09 _step‚1030 /loss/FRt1/assignment_loss‚0.566121008233537 0loss/FRt2/assignment_loss‚0.5081988242371221 .loss/FRt1/detection_loss‚0.514293058232753 0loss/FRt2/detection_loss‚0.49169272657055074 &loss/total_loss‚2.0803056172739627†Vé5Ô2ú- _wandbruntime‚3343  _runtime‚3343š rËûÅ ' lr-AdamW/pg2‚0.00047424193385733426 trainer/global_step‚12549 % _timestamp‚1.7866619948305748e+09 _runtime‚3343.360610292 _step‚1031 ' lr-AdamW/pg1‚0.00047424193385733426ÁÒ²ÙÐüÊ % _timestamp‚1.7866619948305748e+09 _runtime‚3343.360610292 _step‚1031 ' lr-AdamW/pg1‚0.00047424193385733426 ' lr-AdamW/pg2‚0.00047424193385733426 trainer/global_step‚12549‡­(_Î2ý- _wandbruntime‚3343  _runtime‚3343¼% þ»O @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 125829120 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚41348020 @7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚5264 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 122905600 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚345509496609792 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚46135296 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚61478912 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 3034523294720 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚340449700511744 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚340449683472384 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 4654602240 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚46471157 G>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4876 E8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚88597577 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 3034565239808 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚209 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚42130709 BDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 PCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚59049984 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17104896 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 122905600 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚343477947444163 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 5368709120 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4289 PCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚58387380 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚322 % _timestamp‚1.7866619948316376e+09 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚345509555659776 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 3034523294720 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚39467198 UHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚66779136 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 JBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚665 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚4287 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚301744830488576 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚301744763709440 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚42126422 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚298535842152448 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 4628521206 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚322 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚345509555659776 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 3208988336128 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚322 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 5494538240 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚4287 IDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚262 F?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚60 IDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚345509496609792 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 5368709120 KDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚60 trainer/global_step‚12549 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚223 _runtime‚3343.364886652 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚41945088 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚429 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚343477889056783 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17104896 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚46471155 PIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚72’0‰Æ'ÿÀO TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚41945088 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚429 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚343477889056783 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17104896 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚46471155 PIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚72 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 125829120 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚41348020 @7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚5264 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 122905600 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚345509496609792 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚46135296 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚61478912 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 3034523294720 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚340449700511744 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚340449683472384 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 4654602240 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚46471157 G>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4876 E8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚88597577 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 3034565239808 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚209 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚42130709 BDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 PCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚59049984 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17104896 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 122905600 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚343477947444163 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 5368709120 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4289 PCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚58387380 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚322 % _timestamp‚1.7866619948316376e+09 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚345509555659776 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 3034523294720 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚39467198 UHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚66779136 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 JBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚665 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚4287 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚301744830488576 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚301744763709440 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚42126422 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚298535842152448 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 4628521206 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚322 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚345509555659776 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 3208988336128 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚322 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 5494538240 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚4287 IDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚262 F?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚60 IDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚345509496609792 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 5368709120 KDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚60 trainer/global_step‚12549 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚223 _runtime‚3343.364886652ˆºð‹2€- _wandbruntime‚3343  _runtime‚3343Õæ%Ê&ÄM G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚88608712 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚342501295652864 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 3034754363392 NADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚59053056 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚88604417 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 trainer/global_step‚12549 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚46474710 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚88608712 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚340475869265920 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚88604417 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 125829120 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 122905600 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚342501312757760 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 122905600 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚42129709 SFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚66776064 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚343504363810359 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521206 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 3034796311552 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚28934826 B9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚5264 G@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚72 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚10535481 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 3209188444160 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚322 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚298558889525248 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 3028477505079 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 5494538240 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚4293 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 5494538240 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17104896 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚301768124101632 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚61478912 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 125829120 _step‚1033 >5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5264 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚46474708 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 4552982528 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚342501295652864 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚46474708 H@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚665 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚322 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚41948160 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚39470309 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚345536050016256 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 3034754363392 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 5368709120 % _timestamp‚1.7866619950061653e+09 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 4654602240 I@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚4876 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚262 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚342501312757760 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚301768057325568 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 4517085184 E6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚322 G@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚60 D=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚60 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚345536109069312 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚429 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚42134002 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚10535411 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚46132224 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚262 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚4295 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚298558868881408 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚41348068 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 122306526 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 5494538240 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654602240 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 _runtime‚ 3343.53657792 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 3034796311552 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 NGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚70 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 DDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚39470237 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚20643840 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚322 NADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚58387428 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 5368709120 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚41948160 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚42134002 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚28934828 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 845178880 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚340475886305280 K>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59053056 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚262 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚209 IBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚60 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚42129709 EDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚41948160 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚39470309 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚345536050016256 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 3034754363392 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 5368709120 % _timestamp‚1.7866619950061653e+09 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 4654602240 I@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚4876 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚262 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚342501312757760 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚301768057325568 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 4517085184 E6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚322 G@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚60 D=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚60 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚345536109069312 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚429 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚42134002 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚10535411 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚46132224 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚262 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚4295 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚298558868881408 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚41348068 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 122306526 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 5494538240 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654602240 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 _runtime‚ 3343.53657792 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 3034796311552 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 NGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚70 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 DDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚39470237 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚20643840 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚322 NADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚58387428 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 5368709120 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚41948160 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚42134002 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚28934828 QCDeviceStatžI‚sPsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 845178880 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚340475886305280 K>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59053056 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚262 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚209 IBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚60 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚42129709 EDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 trainer/global_step‚12549 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚46474710 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚88608712 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚340475869265920 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚88604417 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 125829120 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 122905600 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚342501312757760 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 122905600 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚42129709 SFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚66776064 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚343504363810359 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521206 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 3034796311552 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚28934826 B9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚5264 G@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚72 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚10535481 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 3209188444160 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚322 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚298558889525248 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 3028477505079 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 5494538240 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚4293 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 5494538240 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17104896 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚301768124101632 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚61478912 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 125829120 _step‚1033 >5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5264 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚46474708 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 4552982528 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚342501295652864‰·…Á2ƒ- _wandbruntime‚3343  _runtime‚3343ˆaÙt„î _step‚1034 0loss/FRt2/assignment_loss‚0.4781344518859833 /loss/FRt1/detection_loss‚0.5531510114244967 trainer/global_step‚12549 % _timestamp‚1.7866619950073495e+09 /loss/FRt1/assignment_loss‚0.659352915551223 /loss/FRt2/detection_loss‚0.4721125211798814 &loss/total_loss‚2.1627509000415843 epoch‚31 _runtime‚3343.537201026ª–*Øy…ó _step‚1034 0loss/FRt2/assignment_loss‚0.4781344518859833 /loss/FRt1/detection_loss‚0.5531510114244967 trainer/global_step‚12549 % _timestamp‚1.7866619950073495e+09 /loss/FRt1/assignment_loss‚0.659352915551223 /loss/FRt2/detection_loss‚0.4721125211798814 &loss/total_loss‚2.1627509000415843 epoch‚31 _runtime‚3343.537201026ŠNLý–†‚(:‹ ì˜ùӜ˩á gpu.0.gpu‚55.0 gpu.0.memory‚11, gpu.0.memoryAllocated‚7.0313262939453125) gpu.0.memoryAllocatedBytes‚ 6039863296 gpu.0.temp‚36.0 gpu.0.powerWatts‚121.059' gpu.0.enforcedPowerLimitWatts‚500.0 gpu.0.powerPercent‚24.2118 gpu.0.smClock‚1410 gpu.0.memoryClock‚1593$ gpu.0.correctedMemoryErrors‚1955# gpu.0.uncorrectedMemoryErrors‚0âŠè)™‡‚(:Ž ì˜ùÓÌ„¨ã disk.sda.in‚30485.58203125 disk.sda2.in‚0" disk.nvme0n1.in‚43757.21484375 disk.dm-1.out‚ 62.0390625$ memory_percent‚68.47248077392578 network.sent‚ 1.6221654e+07 network.recv‚ 6.169278e+06$ disk./.usageGB‚487.1760597229004 disk.sda.out‚ 62.0390625# proc.memory.rssMB‚ 2338.65234375 cpu‚27.20066988607392 disk.drbd2165.out‚0" disk.dm-0.out‚58591.0986328125 disk.sda1.in‚0 disk.sda1.out‚0 disk.dm-1.in‚30485.58203125& proc.memory.availableMB‚ 5165.46875* proc.memory.percent‚14.274001121520996 disk.sda3.in‚30485.58203125 disk.sda3.out‚ 62.0390625$ disk.nvme0c0n1.in‚43757.21484375' disk.nvme0c0n1.out‚58591.0986328125 disk.sda2.out‚0% disk.nvme0n1.out‚58591.0986328125 proc.cpu.threads‚17) disk./.usagePercent‚32.69479413594467 disk.drbd2165.in‚0 disk.dm-0.in‚43757.21484375ë Ú÷2ˆ- _wandbruntime‚3356  _runtime‚3356ê,i½È‰Â $ _timestamp‚1.786662007547329e+09 _runtime‚3356.078062428 _step‚1035 & lr-AdamW/pg1‚0.0004686611783307494 & lr-AdamW/pg2‚0.0004686611783307494 trainer/global_step‚125995ÊÞ,ÍŠÇ trainer/global_step‚12599 $ _timestamp‚1.786662007547329e+09 _runtime‚3356.078062428 _step‚1035 & lr-AdamW/pg1‚0.0004686611783307494 & lr-AdamW/pg2‚0.0004686611783307494‹Ÿ|C½2‹- _wandbruntime‚3356  _runtime‚3356^Þ5RÁ'Œ»O WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 125829120 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 3046152005632 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 5368709120 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚262 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚46648807 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚46648807 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 IBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚68 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 3046152005632 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚39618352 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 5368709120 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17104896 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 845178880 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚346837259315200 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚41945088 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚29042873 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚302909417154560 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚344798798931571 _runtime‚3356.082860919 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚42291524 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚10575413 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚42295811 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17104896 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚39618284 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 5494538240 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 125829120 F?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚60 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚46648805 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚42291524 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 E8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚88940329 G>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4876 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚346837259315200 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚343791090204672 BDeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4289 PCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚58387380 IDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚346837200265216 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚343791090204672 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚20643840 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 125829120 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚322 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 trainer/global_step‚12599 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 122905600 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚88944618 UHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚64681984 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚344798740544191 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 _step‚1036 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚302909481836544 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 4517085184 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚41348020 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 4552982528 M@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚59049984 C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚4289 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚299688190148608 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚61478912 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 122306526 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚322 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 3046110060544 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚10575479 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 5368709120 F>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 D;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚5264 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 5494538240 % _timestamp‚1.7866620075489495e+09=›³Æ'ÀO JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚4287 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 4654602240 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚46648805 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚429 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 3039767395007 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚262 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚343791107309568 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 3221227005952 KDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚60 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚346837200265216 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚343791090204672 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚20643840 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 125829120 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚322 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 trainer/global_step‚12599 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 122905600 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚88944618 UHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚64681984 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚344798740544191 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 _step‚1036 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚302909481836544 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 4517085184 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚41348020 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 4552982528 M@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚59049984 C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚4289 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚299688190148608 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚61478912 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 122306526 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚322 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 3046110060544 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚10575479 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 5368709120 F>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 D;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚5264 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 5494538240 % _timestamp‚1.7866620075489495e+09 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 125829120 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 3046152005632 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 5368709120 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚262 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚46648807 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚46648807 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 IBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚68 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 3046152005632 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚39618352 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 5368709120 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17104896 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 845178880 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚346837259315200 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚41945088 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚29042873 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚302909417154560 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚344798798931571 _runtime‚3356.082860919 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚42291524 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚10575413 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚42295811 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17104896 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚39618284 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 5494538240 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 125829120 F?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚60 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚46648805 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚42291524 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 E8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚88940329 G>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4876 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚346837259315200 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚343791090204672 BDeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4289 PCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚58387380 IDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 _runtime‚3356.243772877 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚429 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚302932775285760 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚299711237521408 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 3039997967619 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚41948160 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚46652358 E8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 BDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚88951464 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚302932712704000 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚343817429647360 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 5494538240 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 122905600 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 3221475182592 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 4517085184 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚262 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 125829120 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚346863812724736 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17104896 G@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚68 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 125829120 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 IBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚60 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 5368709120 >5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5264 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚46652360 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 H@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚665 PDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚41937920 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 5494538240 E6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚322 NADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚59053056 NGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚66 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚344825215297767 NADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚58387428 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚344825156910339 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚4293 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚10576208 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 845178880 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚322 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚39621352 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚39621284 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚10576274 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚209 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚88947169 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 % _timestamp‚1.7866620077134545e+09 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚42294811 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚346863812724736 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚29045076 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚322 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 D=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚60 K>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59053056 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654602240 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17104896 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚343817412542464 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 4552982528 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 831258624 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 3221517120512 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 5494538240 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚4293 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 4552982528 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521206 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚322 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 125829120 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 D6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚322 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 D=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚60 K>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59053056 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654602240 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17104896 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚343817412542464 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 4552982528 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 831258624 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 3221517120512 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 5494538240 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚4293 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 4552982528 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521206 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚322 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 125829120 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 DDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 _runtime‚3356.243772877 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚429 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚302932775285760 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚299711237521408 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 3039997967619 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚41948160 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚46652358 E8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 BDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚88951464 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚302932712704000 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚343817429647360 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 5494538240 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 122905600 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 3221475182592 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 4517085184 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚262 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 125829120 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚346863812724736 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17104896 G@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚68 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 125829120 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 IBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚60 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 5368709120 >5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5264 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚46652360 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 H@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚665 PDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚41937920 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 5494538240 E6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚322 NADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚59053056 NGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚66 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚344825215297767 NADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚58387428 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚344825156910339 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚4293 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚10576208 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 845178880 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚322 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚39621352 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚39621284 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0›•ÚA2‘- _wandbruntime‚3356  _runtime‚3356雩ós’í %loss/total_loss‚2.268237254573015 trainer/global_step‚12599 % _timestamp‚1.7866620077145636e+09 _runtime‚3356.244413258 0loss/FRt1/assignment_loss‚0.6935919609154569 .loss/FRt1/detection_loss‚0.514416675523205 epoch‚31 _step‚1038 0loss/FRt2/assignment_loss‚0.5540957238600047 /loss/FRt2/detection_loss‚0.5061328942743483Ø)wÄx“ò % _timestamp‚1.7866620077145636e+09 _runtime‚3356.244413258 0loss/FRt1/assignment_loss‚0.6935919609154569 .loss/FRt1/detection_loss‚0.514416675523205 epoch‚31 _step‚1038 0loss/FRt2/assignment_loss‚0.5540957238600047 /loss/FRt2/detection_loss‚0.5061328942743483 %loss/total_loss‚2.268237254573015 trainer/global_step‚12599ŽBk•–”‚(:‹ û˜ùÓð¡¡á gpu.0.gpu‚64.0 gpu.0.memory‚12, gpu.0.memoryAllocated‚7.0313262939453125) gpu.0.memoryAllocatedBytes‚ 6039863296 gpu.0.temp‚37.0 gpu.0.powerWatts‚184.427' gpu.0.enforcedPowerLimitWatts‚500.0 gpu.0.powerPercent‚36.8854 gpu.0.smClock‚1410 gpu.0.memoryClock‚1593$ gpu.0.correctedMemoryErrors‚1955# gpu.0.uncorrectedMemoryErrors‚0ûU™,‰•‚(:þ û˜ùÓü¯äâ disk.drbd2165.out‚0 disk.sda2.out‚0 disk.sda.out‚ 62.16015625 disk.sda1.out‚0 disk.sda2.in‚0 disk.dm-1.in‚ 31629.8125 disk.drbd2165.in‚0 disk.dm-0.out‚58597.82421875 disk.sda3.in‚ 31629.8125% disk.nvme0c0n1.out‚58597.82421875 disk.dm-0.in‚ 43769.7890625 disk.sda3.out‚ 62.16015625$ memory_percent‚68.47198009490967) proc.memory.availableMB‚ 5165.55078125# proc.memory.rssMB‚ 2338.65234375* proc.memory.percent‚14.274001121520996 cpu‚27.20016057394997! disk.nvme0n1.in‚ 43769.9140625 network.sent‚ 1.631116e+07) disk./.usagePercent‚32.69485660178302$ disk./.usageGB‚487.1769905090332# disk.nvme0c0n1.in‚ 43769.7890625 disk.sda1.in‚0 disk.dm-1.out‚ 62.16015625 proc.cpu.threads‚17 network.recv‚ 6.195402e+06 disk.sda.in‚ 31629.8125# disk.nvme0n1.out‚58597.82421875 ãÃõ2–- _wandbruntime‚3368  _runtime‚3368$1–WÈ—Â _step‚1039 & lr-AdamW/pg1‚0.0004630984926709559 & lr-AdamW/pg2‚0.0004630984926709559 trainer/global_step‚12649 % _timestamp‚1.7866620199898741e+09 _runtime‚ 3368.51988576À‡å÷Í˜Ç _step‚1039 & lr-AdamW/pg1‚0.0004630984926709559 & lr-AdamW/pg2‚0.0004630984926709559 trainer/global_step‚12649 % _timestamp‚1.7866620199898741e+09 _runtime‚ 3368.51988576ÅONN2™- _wandbruntime‚3368  _runtime‚3368„_@Á'š»O E8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚89283081 @7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚5264 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚345107224199168 IBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚69 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚304074128460288 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚300840558788608 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚343068262825984 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚4287 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚345107207094272 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚345107224199168 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚429 IBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚60 M@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚59049984 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 125829120 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚348164903920640 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 3233549027840 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚346119592031599 IDeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4289 UHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚68876288 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 4552982528 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚39769328 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 G>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4876 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 4654602240 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 5494538240 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚322 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17104896 BDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚42456626 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 5494538240 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚262 PIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚67 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 845178880 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 831258624 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 125829120 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚262 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚42456626 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 5494538240 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 4654602240 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚46826457 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 122306526 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚322 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚348164962970624 IDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 _runtime‚3368.523867605 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚348164903920640 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚46826457 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚42460913 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 _step‚1040 PCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚59049984 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚46826455 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚42460913 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚39769259 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚29153121 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 3233500795392 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 5368709120 C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚4289 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚10616138 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 % _timestamp‚1.7866620199907637e+09 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚348164962970624 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚89287370 JBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚665 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚29153123 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚41945088 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 3057696826368 D;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚5264 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 KDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚60 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 4552982528 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 3057738771456 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 4517085184 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 3051329205615 KBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚4876 PCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚58387380 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚322 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚4287 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚61478912 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 4628521206 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 3057696826368 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚48232448 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 3051370553635 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚20643840 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚46826455 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚346119650418979 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 5368709120 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 122905600 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚345107207094272 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 5368709120 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 3057738771456 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 122905600 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚322 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚343068279865344 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚41348020 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚10616205 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚209 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚300840579432448 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 trainer/global_step‚12649 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚41945088ÎËb›ÀO RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 122905600 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚345107207094272 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 5368709120 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 3057738771456 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 122905600 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚322 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚343068279865344 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚41348020 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚10616205 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚209 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚300840579432448 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 trainer/global_step‚12649 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚41945088 E8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚89283081 @7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚5264 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚345107224199168 IBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚69 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚304074128460288 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚300840558788608 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚343068262825984 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚4287 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚345107207094272 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚345107224199168 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚429 IBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚60 M@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚59049984 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 125829120 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚348164903920640 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 3233549027840 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚346119592031599 IDeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4289 UHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚68876288 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 4552982528 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚39769328 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 G>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4876 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 4654602240 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 5494538240 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚322 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17104896 BDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚4N Óÿd2456626 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 5494538240 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚262 PIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚67 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 845178880 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 831258624 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 125829120 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚262 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚42456626 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 5494538240 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 4654602240 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚46826457 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 122306526 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚322 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚348164962970624 IDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 _runtime‚3368.523867605 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚348164903920640 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚46826457 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚42460913 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 _step‚1040 PCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚59049984 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚46826455 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚42460913 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚39769259 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚29153121 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 3233500795392 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 5368709120 C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚4289 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚10616138 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 % _timestamp‚1.7866620199907637e+09 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚348164962970624 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚89287370 JBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚665 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚29153123 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚41945088 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 3057696826368 D;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚5264 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 KDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚60 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 4552982528 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 3057738771456 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 4517085184 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 3051329205615 KBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚4876 PCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚58387380 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚322 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚4287 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚61478912 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 4628521206 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 3057696826368 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚48232448 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 3051370553635 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚20643840 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚46826455 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚346119650418979 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 5368709120 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0YÖå‘2œ- _wandbruntime‚3368  _runtime‚3368mňèË&ÅM JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚262 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚346146008397747 _step‚1041 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚209 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 125829120 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚42459913 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 125829120 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 H@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚665 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚39772341 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚46132224 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 125829120 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚345133529432064 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 122905600 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚61478912 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚41348068 IBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚60 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚42459913 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 122306526 % _timestamp‚1.7866620201577096e+09 _runtime‚3368.687987247 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚41948160 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚10617013 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚262 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 >5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5264 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚4293 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 3057927895040 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 5368709120 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 5368709120 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚89294216 EDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59053056 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 4654602240 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 122905600 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚42464206 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚429 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚322 PDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚89289921 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚29155326 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 NADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚58387428 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 5368709120 E6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚322 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 4552982528 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 BDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521206 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚322 DDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚39772275 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚343094465658880 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 3051601126295 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚29155328 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚89289921 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚46830008 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚42464206 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚345133546536960 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚348191457327104 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 3057969843200 I@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚4876 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚348191516380160 NADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚59053056 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚300863626805248 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚20643840 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚4295 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚345133529432064 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚46830010 NGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚64 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚10616949 SFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚66776064 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 4517085184 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 3233747235328 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 3051559778227 trainer/global_step‚12649 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654602240 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚322 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚345133546536960 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17104896 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚89294216 B9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚5264 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0_5¹pÐ&žÊM LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚209 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 125829120 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚42459913 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 125829120 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 H@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚665 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚39772341 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚46132224 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 125829120 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚345133529432064 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 122905600 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚61478912 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚41348068 IBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚60 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚42459913 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 122306526 % _timestamp‚1.7866620201577096e+09 _runtime‚3368.687987247 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚41948160 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚10617013 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚262 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 >5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5264 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚4293 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 3057927895040 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 5368709120 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 5368709120 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚89294216 EDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59053056 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 4654602240 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 122905600 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚42464206 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚429 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚322 PDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚89289921 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚29155326 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 NADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚58387428 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 5368709120 E6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚322 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 4552982528 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 BDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521206 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚322 DDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚39772275 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚343094465658880 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 3051601126295 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚29155328 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚89289921 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚46830008 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚42464206 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚345133546536960 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚348191457327104 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 3057969843200 I@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚4876 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚348191516380160 NADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚59053056 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚300863626805248 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚20643840 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚4295 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚345133529432064 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚46830010 NGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚64 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚10616949 SFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚66776064 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 4517085184 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 3233747235328 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 3051559778227 trainer/global_step‚12649 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654602240 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚322 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚345133546536960 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17104896 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚89294216 B9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚5264 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚262 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚346146008397747 _step‚1041‘-¡|Û2Ÿ- _wandbruntime‚3368  _runtime‚3368‰ºsu ï 0loss/FRt1/assignment_loss‚0.6353611279991981 /loss/FRt1/detection_loss‚0.5340001972253628 0loss/FRt2/detection_loss‚0.49409596962595775 % _timestamp‚1.7866620201588857e+09 0loss/FRt2/assignment_loss‚0.5117509479528574 %loss/total_loss‚2.175208242803376 epoch‚31 trainer/global_step‚12649 _runtime‚3368.688777921 _step‚1042 ÙÚ‚z¡ô 0loss/FRt1/assignment_loss‚0.6353611279991981 /loss/FRt1/detection_loss‚0.5340001972253628 0loss/FRt2/detection_loss‚0.49409596962595775 % _timestamp‚1.7866620201588857e+09 0loss/FRt2/assignment_loss‚0.5117509479528574 %loss/total_loss‚2.175208242803376 epoch‚31 trainer/global_step‚12649 _runtime‚3368.688777921 _step‚1042’Tãu)•¢‚(:Š Š™ùÓ숛á gpu.0.gpu‚61.0 gpu.0.memory‚11, gpu.0.memoryAllocated‚7.0313262939453125) gpu.0.memoryAllocatedBytes‚ 6039863296 gpu.0.temp‚36.0 gpu.0.powerWatts‚126.695' gpu.0.enforcedPowerLimitWatts‚500.0 gpu.0.powerPercent‚25.339 gpu.0.smClock‚1410 gpu.0.memoryClock‚1593$ gpu.0.correctedMemoryErrors‚1955# gpu.0.uncorrectedMemoryErrors‚0UpAX|£‚(:ñ Š™ùÓ›£Óã# proc.memory.rssMB‚ 2338.65234375 proc.cpu.threads‚17( disk.nvme0c0n1.out‚58647.25048828125 network.sent‚ 1.638166e+07) disk./.usagePercent‚32.69904437302813 disk.drbd2165.out‚0 disk.dm-0.in‚ 43782.09375! disk.nvme0c0n1.in‚ 43782.09375 disk.sda.out‚62.4375 disk.sda1.in‚0 disk.drbd2165.in‚0 disk.sda3.out‚62.4375 disk.sda1.out‚0 disk.sda2.in‚0 disk.sda2.out‚0& disk.nvme0n1.out‚58647.25048828125* proc.memory.percent‚14.274001121520996 cpu‚27.1998524557675 disk.dm-1.out‚62.4375# disk.dm-0.out‚58647.25048828125 disk.sda3.in‚ 31629.8125 disk.nvme0n1.in‚ 43782.09375# memory_percent‚68.4718132019043 network.recv‚ 6.223285e+06$ disk./.usageGB‚487.2393913269043 disk.dm-1.in‚ 31629.8125 disk.sda.in‚ 31629.8125' proc.memory.availableMB‚ 5165.578125ï­ŸŽ2¤- _wandbruntime‚3381  _runtime‚3381dlmÅ¥¿ % lr-AdamW/pg1‚0.000457554234158859 % lr-AdamW/pg2‚0.000457554234158859 trainer/global_step‚12699 # _timestamp‚1.78666203301175e+09 _runtime‚3381.541819942 _step‚1043˜ï/Ê¦Ä trainer/global_step‚12699 # _timestamp‚1.78666203301175e+09 _runtime‚3381.541819942 _step‚1043 % lr-AdamW/pg1‚0.000457554234158859 % lr-AdamW/pg2‚0.000457554234158859“›ÚÄ2§- _wandbruntime‚3381  _runtime‚3381„8H¨ºO GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚89630122 BDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4876 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 4654602240 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 5368709120 M@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚59049984 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚41945088 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 5368709120 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚42626015 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚41945088 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚262 KDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚60 @7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚5264 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚349492607576064 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚42626015 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 3069325537280 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚10656643 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 125829120 KBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚4876 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 831258624 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 F>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 PCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚59049984 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚429 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 3245833944576 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚347440443519007 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚29263373 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚41348020 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚223 PIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚74 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚347440501906387 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚322 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚39920014 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚61478912 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚322 C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚4289 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 3069283592192 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 =6DeviceStatsMonitgÁ› xor.on_train_batch_start/max_split_size‚-1 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 3062891016223 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 5494538240 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 125829120 trainer/global_step‚12699 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚346423323983872 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 122905600 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚346423323983872 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚47004105 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚344377552502784 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 5494538240 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 125829120 JBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚665 IBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚76 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 _step‚1044 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚42621728 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚346423341088768 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚41940992 E8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚89625833 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚349492607576064 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 4552982528 IBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚60 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 3069325537280 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 _runtime‚3381.546919822 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚47004107 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚4287 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚322 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 3069283592192 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚39920090 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚305238719432192 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 845178880 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 3062932364243 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 5494538240 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚47004107 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4289 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17104896 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚305238782017024 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 4517085184 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚322 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚29263371 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 4552982528 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚209 UHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚62584832 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 3245792003584 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚262 F?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚60 $ _timestamp‚1.786662033012779e+09 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚4287 PCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚58387380 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 4628521206 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 122306526 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚262 IDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 PCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚59049984 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚429 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 3245833944576 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚347440443519007 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚29263373 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚41348020 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚223 PIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚74 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚347440501906387 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚322 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚39920014 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚61478912 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚322 C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚4289 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 3069283592192 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 3062891016223 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 5494538240 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 125829120 trainer/global_step‚12699 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚346423323983872 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 122905600 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚346423323983872 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚47004105 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚344377552502784 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 5494538240 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 125829120 JBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚665 IBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚76 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 _step‚1044 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚42621728 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚346423341088768 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚41940992 E8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚89625833 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚349492607576064 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 4552982528 IBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚60 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 3069325537280 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 _runtime‚3381.546919822 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚47004107 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚4287 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚322 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 3069283592192 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚39920090 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚305238719432192 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 845178880 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 3062932364243 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 5494538240 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚47004107 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4289 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17104896 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚305238782017024 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 4517085184 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚322 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚29263371 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 4552982528 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚209 UHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚62584832 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 3245792003584 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚262 F?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚60 $ _timestamp‚1.786662033012779e+09 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚4287 PCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚58387380 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 4628521206 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 122306526 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚262 IDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4876 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 4654602240 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 5368709120 M@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚59049984 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚41945088 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 5368709120 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚42626015 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚41945088 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚262 KDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚60 @7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚5264 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚349492607576064 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚42626015 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 3069325537280 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚10656643 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 125829120”.Jº2ª- _wandbruntime‚3381  _runtime‚3381fÑŸË&«ÅM J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚47007658 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 4654602240 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚346449663426560 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚89632673 B9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚5264 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚302015974801408 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚347466918272583 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 125829120 K>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59053056 _step‚1045 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚42625015 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚346449663426560 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 4552982528 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚44035072 BDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521206 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚39923044 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 3069556609024 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚41948160 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚89636968 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 122905600 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 125829120 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 3069514660864 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 5494538240 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 122905600 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 3069556609024 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚209 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 5494538240 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚322 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 3246034740736 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 D=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚60 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17104896 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚223 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚262 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654602240 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚10657468 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚349519220035584 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 % _timestamp‚1.7866620331792114e+09 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17104896 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 831258624 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚322 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚47007658 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 3063162936903 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 4517085184 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚41348068 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚89636968 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚42629308 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 3069514660864 NADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚58387428 G@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚64 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚344403738296320 _runtime‚3381.709655063 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚349519220035584 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚41948160 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚42629308 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚42625015 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚429 SFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚64678912 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 3063121588835 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 122306526 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚4293 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚346449646321664 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 5368709120 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 5368709120 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚322 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚262 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚4295 PDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚29265576 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚305262074221056 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚61478912 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 EDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚322 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚262 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚89632673 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚344403755335680 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚39923108 >5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5264 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚346449646321664 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚349519160982528 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚20643840 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 E6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚322 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚262 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚4295 PDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚29265576 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚305262074221056 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚61478912 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 EDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚322 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚262 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚89632673 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚344403755335680 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚39923108 >5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5264 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚346449646321664 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚349519160982528 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚20643840 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 EDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59053056 _step‚1045 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚42625015 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚346449663426560 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 4552982528 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚44035072 BDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521206 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚39923044 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 3069556609024 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚41948160 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚89636968 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 122905600 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 125829120 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 UÛ-uü- CDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 3069514660864 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 5494538240 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 122905600 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 3069556609024 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚209 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 5494538240 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚322 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 3246034740736 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 D=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚60 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17104896 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚223 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚262 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654602240 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚10657468 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚349519220035584 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 % _timestamp‚1.7866620331792114e+09 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17104896 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 831258624 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚322 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚47007658 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 3063162936903 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 4517085184 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚41348068 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚89636968 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚42629308 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 3069514660864 NADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚58387428 G@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚64 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚344403738296320 _runtime‚3381.709655063•âï  2­- _wandbruntime‚3381  _runtime‚3381MîGØu®ï /loss/FRt1/assignment_loss‚0.556303273563537 1loss/FRt2/assignment_loss‚0.46620048235807526 0loss/FRt2/detection_loss‚0.47114303767751564 &loss/total_loss‚1.9624281933952734 % _timestamp‚1.7866620331804445e+09 _runtime‚ 3381.71030953 _step‚1046 /loss/FRt1/detection_loss‚0.4687813997961456 epoch‚31 trainer/global_step‚12699v¦4z¯ô /loss/FRt1/assignment_loss‚0.556303273563537 1loss/FRt2/assignment_loss‚0.46620048235807526 0loss/FRt2/detection_loss‚0.47114303767751564 &loss/total_loss‚1.9624281933952734 % _timestamp‚1.7866620331804445e+09 _runtime‚ 3381.71030953 _step‚1046 /loss/FRt1/detection_loss‚0.4687813997961456 epoch‚31 trainer/global_step‚12699–¢‰d“–°‚(:‹ ™™ùÓ©©á gpu.0.gpu‚64.0 gpu.0.memory‚12, gpu.0.memoryAllocated‚7.0313262939453125) gpu.0.memoryAllocatedBytes‚ 6039863296 gpu.0.temp‚37.0 gpu.0.powerWatts‚217.416' gpu.0.enforcedPowerLimitWatts‚500.0 gpu.0.powerPercent‚43.4832 gpu.0.smClock‚1410 gpu.0.memoryClock‚1593$ gpu.0.correctedMemoryErrors‚1955# gpu.0.uncorrectedMemoryErrors‚0E±%•±‚(:„ ™™ùÓκˆä( disk.nvme0c0n1.out‚58653.11572265625 disk.sda.in‚ 31629.8125 disk.sda.out‚ 62.55078125 disk.sda1.in‚0& disk.nvme0n1.out‚58653.11572265625) proc.memory.availableMB‚ 5165.72265625# proc.memory.rssMB‚ 2338.65234375% disk./.usageGB‚487.23940658569336 disk.dm-0.in‚ 43782.09375 disk.sda3.out‚ 62.55078125 disk.sda2.out‚0 disk.dm-1.out‚ 62.55078125 disk.drbd2165.in‚0 disk.drbd2165.out‚0 cpu‚27.199408731756748! disk.nvme0c0n1.in‚ 43782.09375 disk.sda2.in‚0 disk.nvme0n1.in‚ 43782.09375 proc.cpu.threads‚17 network.sent‚ 1.6452746e+07) disk./.usagePercent‚32.69904539705826# disk.dm-0.out‚58653.11572265625 disk.sda3.in‚ 31629.8125 disk.sda1.out‚0 disk.dm-1.in‚ 31629.8125$ memory_percent‚68.47093105316162* proc.memory.percent‚14.274001121520996 network.recv‚ 6.251299e+065 3^2²- _wandbruntime‚3394  _runtime‚3394núµû˳Š_runtime‚3394.339691484 _step‚1047 ' lr-AdamW/pg1‚0.00045202875889182226 ' lr-AdamW/pg2‚0.00045202875889182226 trainer/global_step‚12749 % _timestamp‚1.7866620458096263e+09™ùÐ´Ê ' lr-AdamW/pg1‚0.00045202875889182226 ' lr-AdamW/pg2‚0.00045202875889182226 trainer/global_step‚12749 % _timestamp‚1.7866620458096263e+09 _runtime‚3394.339691484 _step‚1047—ÎÈYD2µ- _wandbruntime‚3394  _runtime‚3394:¹©¿'¶¹O JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚4287 M@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚59049984 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚350820370281472 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚4287 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 G>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4876 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚347739440873472 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚4289 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚40071209 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚10697586 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 831258624 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚61478912 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 122905600 PCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚58387380 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚10697510 BDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 @7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚5264 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚89972874 D;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚5264 IBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚78 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 122905600 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚350820311231488 E8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚89968585 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 125829120 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚345686859218944 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 5368709120 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 4654602240 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 4552982528 UHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚66779136 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 3258128670720 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 5368709120 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚42791117 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4289 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚47181757 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 4628521206 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 3080870358016 JBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚665 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚306403378604032 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚20643840 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 125829120 _step‚1048 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 4654602240 PIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚76 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚322 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚41348020 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 5494538240 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚262 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚348761295006415 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚322 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚303145296068608 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚345686842179584 trainer/global_step‚12749 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 3080870358016 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 122306526 KBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚4876 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚47181755 F>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚42786830 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚350820370281472 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17104896 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 3080912303104 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚41945088 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚347739457978368 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 3074494174851 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚347739457978368 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚42791117 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚40071131 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 845178880 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚303145316712448 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚262 IBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚60 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚350820311231488 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 4552982528 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚429 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚29373621 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 3258082535424 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 4517085184 _runtime‚3394.343885761 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚322 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 F?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚60 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 5494538240KM[pÄ'·¾O ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚348761353393795 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 5494538240 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚42786830 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚29373623 IDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 @7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚5264 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚89972874 D;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚5264 IBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚78 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 122905600 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚350820311231488 E8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚89968585 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 125829120 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚345686859218944 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 5368709120 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 4654602240 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 4552982528 UHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚66779136 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 3258128670720 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 5368709120 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚42791117 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4289 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚47181757 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 4628521206 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 3080870358016 JBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚665 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚306403378604032 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚20643840 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 125829120 _step‚1048 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 4654602240 PIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚76 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚322 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚41348020 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 5494538240 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚262 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚348761295006415 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚322 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚303145296068608 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚345686842179584 trainer/global_step‚12749 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 3080870358016 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 122306526 KBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚4876 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚47181755 F>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚42786830 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚350820370281472 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17104896 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 3080912303104 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚41945088 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚347739457978368 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 3074494174851 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚347739457978368 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚42791117 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚40071131 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 845178880 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚303145316712448 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚262 IBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚60 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚350820311231488 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 4552982528 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚429 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚29373621 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 3258082535424 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 4517085184 _runtime‚3394.343885761 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚322 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 F?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚60 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 5494538240 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚4287 M@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚59049984 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚350820370281472 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚4287 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 G>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4876 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚347739440873472 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚4289 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚40071209 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚10697586 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 831258624 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚61478912 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 122905600 PCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚58387380 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚10697510 B6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚322 G@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚60 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 EDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59053056 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚347765780316160 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚89975425 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚223 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 5494538240 NADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚59053056 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚348787711372563 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚262 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚41948160 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521206 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚47185310 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚42794410 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 3081143374848 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚345713027973120 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚429 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚350846864637952 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 3081143374848 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚89979720 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 3258327894016 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 4654602240 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚29375828 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 122306526 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚350846923691008 >5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5264 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚10698426 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚345713045012480 # _timestamp‚1.78666204597749e+09 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 4552982528 D=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚60 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚42790117 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 3081101426688 EDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚40074178 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 845178880 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 5494538240 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 5368709120 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚322 _runtime‚ 3394.50797871 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚4293 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚347765763211264 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 125829120 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚350846923691008 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚347765780316160 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚41348068 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚89975425 PDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚42790117 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚29375826 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚348787769759991 NADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚58387428 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.lqtµRQ arge_pool.peak‚ 4517085184 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 125829120 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 3081101426688 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚10698352 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚306426671335424 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚262 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚262 D6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚322 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 trainer/global_step‚12749 NGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚74 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚303168364085248 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚303168343441408 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 5494538240 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654602240 B9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚5264 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 3258371929088 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 IBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚60 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚89979720 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚47185308 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 H@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚665 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 3074683399443 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 125829120EdÂYÍ&ºÇM ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚347765763211264 BDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚42790117 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚29375826 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚348787769759991 NADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚58387428 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 4517085184 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 125829120 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 3081101426688 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚10698352 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚306426671335424 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚262 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚262 D6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚322 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 trainer/global_step‚12749 NGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚74 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚303168364085248 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚303168343441408 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 5494538240 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654602240 B9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚5264 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 3258371929088 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 IBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚60 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚89979720 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚47185308 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 H@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚665 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 3074683399443 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 125829120 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚4293 G@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚76 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚322 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚322 G@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚60 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 EDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59053056 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚347765780316160 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚89975425 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚223 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 5494538240 NADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚59053056 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚348787711372563 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚262 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚41948160 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521206 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚47185310 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚42794410 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 3081143374848 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚345713027973120 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚429 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚350846864637952 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 3081143374848 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚89979720 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 3258327894016 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 4654602240 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚29375828 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 122306526 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚350846923691008 >5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5264 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚10698426 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚345713045012480 # _timestamp‚1.78666204597749e+09 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 4552982528 D=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚60 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚42790117 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 3081101426688 EDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚40074178 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 845178880 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 5494538240 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 5368709120 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚322 _runtime‚ 3394.50797871 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚4293 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚347765763211264 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 125829120 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚350846923691008 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚347765780316160 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚41348068 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚89975425 PDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚352148073936896 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚349055574867968 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚47359407 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚42951932 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚429 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚44038144 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 KDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚60 IBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚76 PCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚58387380 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 125829120 trainer/global_step‚12799 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚349055574867968 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 3092499068928 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚350082146493823 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚262 @7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚5264 F>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 845178880 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚42956219 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚307568108011520 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 122306526 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 5368709120 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚262 C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚4289 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚40222400 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚10738527 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚304297664708608 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 3270422659072 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚61478912 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚42951932 G>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4876 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚41348020 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚322 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚41945088 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚29483871 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚346996148895744 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚346996131856384 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 5368709120 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 125829120 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 3092457123840 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 831258624 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17104896 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚322 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚4287 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 122905600 KBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚4876 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚29483873 PIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚74 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 4654602240 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚352148014886912 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚20643840 BDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚352148014886912 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 3092499068928 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 3092457123840 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚322 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 5494538240 D;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚5264 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 4517085184 M@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚59049984 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4289 JBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚665 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚223 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 % _timestamp‚1.7866620586254342e+09 _runtime‚3407.158880975 IDeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4289 JBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚665 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚223 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 % _timestamp‚1.7866620586254342e+09 _runtime‚3407.158880975 IDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚352148073936896 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚349055574867968 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚47359407 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚42951932 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚429 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚44038144 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 KDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚60 IBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚76 PCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚58387380 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 125829120 trainer/global_step‚12799 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚349055574867968 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 3092499068928 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚350082146493823 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚262 @7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚5264 F>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 845178880 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚42956219 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚307568108011520 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 122306526 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 5368709120 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚262 C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚4289 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚40222400 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚10738527 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚304297664708608 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 3270422659072 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚61478912 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚42951932 G>DeviceStatsMonitor.on_train_batÒ£9: ch_start/active.small_pool.peak‚4876 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚41348020 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚322 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚41945088 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚29483871 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚346996148895744 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚346996131856384 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 5368709120 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 125829120 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 3092457123840 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 831258624 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17104896 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚322 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚4287 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 122905600 KBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚4876 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚29483873 PIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚74 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 4654602240 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚352148014886912 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚20643840 BDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚352148014886912 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 3092499068928 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 3092457123840 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚322 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 5494538240œ_£ÁN2Æ- _wandbruntime‚3407  _runtime‚3407zLÀË&ÇÅM SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 5368709120 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚90318177 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚42959512 PDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59053056 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚322 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚350108621247399 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚47362960 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 831258624 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚322 trainer/global_step‚12799 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚41348068 NADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚59053056 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 122905600 E8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚47362960 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚41948160 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚42959512 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚350108562859971 % _timestamp‚1.7866620587877877e+09 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚29486078 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 125829120 G@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚66 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚40225338 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚307591399691264 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 125829120 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 4552982528 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚349081880100864 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚90322472 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 4552982528 _step‚1053 >5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5264 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚47362958 I@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚4876 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 3270666966016 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 3270622930944 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚262 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚4293 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚44035072 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚322 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚349081897205760 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17104896 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 3092688192512 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 4517085184 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚40225404 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚10739326 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚61478912 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚352174627346432 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚42955219 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚262 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚4295 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚42955219 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17104896 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 3092730140672 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚90322472 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚29486076 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚307591335012352 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 D=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚60 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 3086286558119 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 5368709120 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654602240 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚352174568293376 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 3092688192512 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚304320712081408 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521206 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 5494538240 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 4654602240 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚223 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 5494538240 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 5494538240 _runtime‚3407.318065707 EDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚4293 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 3092730140672 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 3086245210051 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚47362958 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 122905600 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚304320732725248 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚20643840 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 B8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚47362960 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚41948160 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚42959512 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚350108562859971 % _timestamp‚1.7866620587877877e+09 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚29486078 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 125829120 G@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚66 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚40225338 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚307591399691264 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 125829120 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 4552982528 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚349081880100864 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚90322472 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 4552982528 _step‚1053 >5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5264 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚47362958 I@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚4876 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 3270666966016 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 3270622930944 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚262 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚4293 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚44035072 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚322 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚349081897205760 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17104896 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 3092688192512 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 4517085184 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚40225404 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚10739326 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚61478912 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚352174627346432 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚42955219 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚262 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚4295 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚42955219 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17104896 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 3092730140672 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚90322472 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚29486076 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚307591335012352 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 D=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚60 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 3086286558119 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 5368709120 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654602240 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚352174568293376 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 3092688192512 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚304320712081408 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521206 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 5494538240 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 4654602240 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚223 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 5494538240 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 5494538240 _runtime‚3407.318065707 EDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚4293 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 3092730140672 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 3086245210051 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚47362958 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 122905600 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚304320732725248 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚20643840 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 BDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59053056 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚322ºLõ2É- _wandbruntime‚3407  _runtime‚3407`†auÊï 0loss/FRt1/assignment_loss‚0.5765275936354827 1loss/FRt2/assignment_loss‚0.47578698811242637 /loss/FRt1/detection_loss‚0.5143949406957888 epoch‚31 % _timestamp‚1.7866620587887282e+09 0loss/FRt2/detection_loss‚0.49347652327328184 $loss/total_loss‚2.06018604571698 trainer/global_step‚12799 _runtime‚3407.318532651 _step‚1054±Ù6zËô 0loss/FRt1/assignment_loss‚0.5765275936354827 1loss/FRt2/assignment_loss‚0.47578698811242637 /loss/FRt1/detection_loss‚0.5143949406957888 epoch‚31 % _timestamp‚1.7866620587887282e+09 0loss/FRt2/detection_loss‚0.49347652327328184 $loss/total_loss‚2.06018604571698 trainer/global_step‚12799 _runtime‚3407.318532651 _step‚1054žiÆ÷2Ì- _wandbruntime‚3412  _runtime‚3412ÆŠˆÆÞÍØ /Purity/*FRt/event_purity‚0.4770564138889313 Purity/0FRt/event_purity‚1 3Purity/1FRt/event_proportion‚0.6297432780265808 +jet/accuracy_2_of_2‚0.22442132234573364 particle/accuracy_1_of_1‚1 (particle/accuracy‚0.6411365270614624 /Purity/2FRt/event_purity‚0.4527990221977234 +particle/sensitivity‚0.6400243043899536 trainer/global_step‚12799 )loss/val_total_loss‚2.382628465890715 /particle/accuracy_2_of_2‚0.6977397203445435 2Purity/2FRt/event_proportion‚0.040062315762043 *jet/accuracy_1_of_2‚0.5982743501663208 +validation_accuracy‚0.22442132234573364 $ _timestamp‚1.786662063983261e+09 _step‚1055 -Purity/2FRt/FRt_purity‚0.5867030620574951 /particle/accuracy_1_of_2‚0.9159412384033203 7validation_average_jet_accuracy‚0.32377538084983826 _runtime‚3412.513262607 2Purity/*FRt/event_proportion‚0.669805645942688 .Purity/*FRt/FRt_purity‚0.49083924293518066 -Purity/1FRt/FRt_purity‚0.4785241484642029 'particle/f_score‚0.5587037801742554 epoch‚31 3Purity/0FRt/event_proportion‚0.3301943838596344 /Purity/1FRt/event_purity‚0.4785241484642029 *jet/accuracy_1_of_1‚0.3181581497192383 +particle/specificity‚0.6417458653450012XÃÌ»ãÎÝ epoch‚31 3Purity/0FRt/event_proportion‚0.3301943838596344 /Purity/1FRt/event_purity‚0.4785241484642029 *jet/accuracy_1_of_1‚0.3181581497192383 +particle/specificity‚0.6417458653450012 /Purity/*FRt/event_purity‚0.4770564138889313 Purity/0FRt/event_purity‚1 3Purity/1FRt/event_proportion‚0.6297432780265808 +jet/accuracy_2_of_2‚0.22442132234573364 particle/accuracy_1_of_1‚1 (particle/accuracy‚0.6411365270614624 /Purity/2FRt/event_purity‚0.4527990221977234 +particle/sensitivity‚0.6400243043899536 trainer/global_step‚12799 )loss/val_total_loss‚2.382628465890715 /particle/accuracy_2_of_2‚0.6977397203445435 2Purity/2FRt/event_proportion‚0.040062315762043 *jet/accuracy_1_of_2‚0.5982743501663208 +validation_accuracy‚0.22442132234573364 $ _timestamp‚1.786662063983261e+09 _step‚1055 -Purity/2FRt/FRt_purity‚0.5867030620574951 /particle/accuracy_1_of_2‚0.9159412384033203 7validation_average_jet_accuracy‚0.32377538084983826 _runtime‚3412.513262607 2Purity/*FRt/event_proportion‚0.669805645942688 .Purity/*FRt/FRt_purity‚0.49083924293518066 -Purity/1FRt/FRt_purity‚0.4785241484642029 'particle/f_score‚0.5587037801742554ŸžŠmå–Ï‚(:‹ ·™ùÓÕ‘á gpu.0.gpu‚61.0 gpu.0.memory‚11, gpu.0.memoryAllocated‚7.0313262939453125) gpu.0.memoryAllocatedBytes‚ 6039863296 gpu.0.temp‚37.0 gpu.0.powerWatts‚211.007' gpu.0.enforcedPowerLimitWatts‚500.0 gpu.0.powerPercent‚42.2014 gpu.0.smClock‚1410 gpu.0.memoryClock‚1593$ gpu.0.correctedMemoryErrors‚1955# gpu.0.uncorrectedMemoryErrors‚0–FÄc}Ђ(:ò ·™ùÓèÿÛã network.sent‚ 1.6602312e+07 disk.drbd2165.in‚0 disk.sda3.in‚ 31629.8125 disk.sda1.in‚0* proc.memory.percent‚14.417052268981934 cpu‚27.2078081334645 network.recv‚ 6.316242e+06) disk./.usagePercent‚32.69983902041441" disk.nvme0n1.in‚43782.20703125 disk.dm-0.in‚43782.20703125$ disk.nvme0c0n1.in‚43782.20703125 disk.sda1.out‚0 disk.sda2.in‚0# proc.memory.rssMB‚ 2362.08984375$ disk./.usageGB‚487.2512321472168$ disk.nvme0n1.out‚58705.560546875! disk.dm-0.out‚58705.560546875& disk.nvme0c0n1.out‚58705.560546875 disk.sda.in‚ 31629.8125 disk.sda2.out‚0$ memory_percent‚68.61827373504639 disk.dm-1.in‚ 31629.8125 disk.dm-1.out‚63.25 disk.drbd2165.out‚0 disk.sda3.out‚63.25 disk.sda.out‚63.25) proc.memory.availableMB‚ 5141.58203125 proc.cpu.threads‚17VÁF,2Ñ- _wandbruntime‚3424  _runtime‚3424ᣮÉÒà _runtime‚3424.896409699 _step‚1056 & lr-AdamW/pg1‚0.0004410355764275291 & lr-AdamW/pg2‚0.0004410355764275291 trainer/global_step‚12849 % _timestamp‚1.7866620763663683e+09¿Ó„ŽÎÓÈ & lr-AdamW/pg1‚0.0004410355764275291 & lr-AdamW/pg2‚0.0004410355764275291 trainer/global_step‚12849 % _timestamp‚1.7866620763663683e+09 _runtime‚3424.896409699 _step‚1056 ÊXíó2Ô- _wandbruntime‚3424  _runtime‚3424Âv‘`Õ»O G>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4876 JBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚665 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 5494538240 _step‚1057 M@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚59049984 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚350762611867648 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 3286753800192 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚322 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4289 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚40423081 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚61478912 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 4628521206 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 5494538240 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 IBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚73 BDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚353870723481088 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 4654602240 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚29634737 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 845178880 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚351788021832552 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 3101642420148 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚322 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚350762628972544 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17104896 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17104896 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚322 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚262 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚43139221 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚309036122138624 WIDeviceStatsMonitor.on_train_batch_start/reservexgÚ)ad_bytes.small_pool.current‚ 125829120 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚4289 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚43139221 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 122306526 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 122905600 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚4287 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚348686321025024 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚41348020 IDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 5368709120 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚262 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚353870723481088 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 3101683768168 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 % _timestamp‚1.7866620763673453e+09 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚47597661 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚41945088 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚305749324300288 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚20643840 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 125829120 PCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚59049984 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚350762628972544 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚40423154 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚429 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 5494538240 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚44038144 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 125829120 trainer/global_step‚12849 @7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚5264 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚43143508 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 4654602240 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 3108153558528 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 4517085184 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚47597663 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 4552982528 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 3108111613440 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚29634735 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚350762611867648 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 3108111613440 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 PCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚58387380 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚262 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 F>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚223 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚351787963445172 D;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚5264 E8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚90736882 IDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4876 JBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚665 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 5494538240 _step‚1057 M@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚59049984 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚350762611867648 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 3286753800192 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚322 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4289 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚40423081 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚61478912 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 4628521206 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 5494538240 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 IBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚73 BDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚353870723481088 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 4654602240 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚29634737 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 845178880 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚351788021832552 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 3101642420148 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚322 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚350762628972544 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17104896 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17104896 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚322 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚262 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚43139221 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚309036122138624 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 125829120 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚4289 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚43139221 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 122306526 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 122905600 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚4287 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚348686321025024 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚41348020 IDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 5368709120 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚262 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚353870723481088 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 3101683768168 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 % _timestamp‚1.7866620763673453e+09 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚47597661 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚41945088 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚305749324300288 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚20643840 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 125829120 PCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚59049984 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚350762628972544 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚40423154 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚429 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 5494538240 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚44038144 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 125829120 trainer/global_step‚12849 @7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚5264 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚43143508 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 4654602240 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 3108153558528 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 4517085184 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚47597663 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 4552982528 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 3108111613440 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚29634735 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚350762611867648 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 3108111613440 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 PCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚58387380 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚262 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 F>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚223 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚351787963445172 D;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚5264 E8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚90736882 IDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 % _timestamp‚1.7866620765450678e+09 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚305772371673088 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 125829120 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚322 G@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚60 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 3108342682112 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 845178880 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚20643840 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 trainer/global_step‚12849 _runtime‚3425.075483873 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚429 BDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚4293 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚350788934205440 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚41348068 IBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚60 P6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚322 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 I@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚4876 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654602240 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚43146801 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚348712523857920 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚41948160 NADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚59053056 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚350788951310336 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 SFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚68873216 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚351814438198748 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 5368709120 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚348712506818560 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 125829120 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚90748017 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17104896 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚43146801 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚90743722 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚43142508 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 3101914340828 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 3101872992760 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 122306526 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 125829120 H@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚665 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚353897335940608 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17104896 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 3108342682112 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 122905600 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚47601214 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚48229376 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 122905600 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 4552982528 NGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚74 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚309059348615168 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚351814379811320 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚262 EDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521206 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚10789176 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚309059417488384 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 4517085184 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 5494538240 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚90748017 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 5494538240 D8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 K>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59053056 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚47601216 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚40426192 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚40426116 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚353897335940608 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚41948160 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚90743722 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚209 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚322 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚262 >5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5264 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚29636940 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚4293 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚305772351029248 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚262 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚4295 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚47601214 G@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚76 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 5368709120 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚29636942 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚350788934205440 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 831258624 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚61478912öV¡'¹ÙÊM \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 3108384630272 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 3286997585920 NADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚58387428 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 4552982528 _step‚1058 EDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚4293 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚350788934205440 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚41348068 IBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚60 P6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚322 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 I@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚4876 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654602240 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚43146801 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚348712523857920 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚41948160 NADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚59053056 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚350788951310336 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 SFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚68873216 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚351814438198748 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 5368709120 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚348712506818560 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 125829120 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚90748017 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17104896 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚43146801 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚90743722 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚43142508 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 3101914340828 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 3101872992760 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 122306526 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 125829120 H@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚665 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚353897335940608 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17104896 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 3108342682112 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 122905600 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚47601214 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚48229376 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 122905600 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 4552982528 NGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚74 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚309059348615168 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚351814379811320 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚262 EDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521206 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚10789176 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚309059417488384 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 4517085184 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 5494538240 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚90748017 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 5494538240 D8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 K>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59053056 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚47601216 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚40426192 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚40426116 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚353897335940608 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚41948160 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚90743722 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚209 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚322 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚262 >5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5264 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚29636940 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚4293 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚305772351029248 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚262 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚4295 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚47601214 G@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚76 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 5368709120 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚29636942 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚350788934205440 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 831258624 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚61478912 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 % _timestamp‚1.7866620765450678e+09 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚305772371673088 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 125829120 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚322 G@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚60 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 3108342682112 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 845178880 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚20643840 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 trainer/global_step‚12849 _runtime‚3425.075483873 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚429 BDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4876 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚322 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚322 $ _timestamp‚1.786662089432881e+09 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚47775311 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚306901672296448 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚43304323 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚355198427136512 UHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚70973440 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚4287 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4289 D;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚5264 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚10829008 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 PCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚58387380 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 3119698379264 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚322 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 4517085184 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 122905600 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚352078728757248 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚209 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 4552982528 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 125829120 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚352078745862144 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚43304323 PIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚71 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚61478912 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 5494538240 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚47775313 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 3119698379264 IDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 5368709120 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 125829120 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚349995627741184 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 5368709120 F>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 4552982528 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 3113245578776 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚41348020 KBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚4876 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 3299066898432 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚355198486186496 JBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚665 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚353108873319960 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 125829120 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚50329600 _step‚1061 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚355198427136512 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 845178880 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚262 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚43308610 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 4654602240 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚355198486186496 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 3119740324352 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17104896 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚91083923 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 _runtime‚3437.966990336 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 3113204230756 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 5368709120 KDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚60 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 5494538240 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 122905600 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚310200688865280 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 E8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚91079634 M@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚59049984 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚47775311 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚4287 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚429 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚29744985 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚353108814932580 trainer/global_step‚12899 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚41945088 IBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚73 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚306901692940288 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 4654602240 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚310200759838720 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚20643840 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚349995610701824 PCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚59049984 IDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 5368709120 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 125829120 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚349995627741184 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 5368709120 F>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 4552982528 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 3113245578776 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚41348020 KBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚4876 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 3299066898432 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚355198486186496 JBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚665 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚353108873319960 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 125829120 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚50329600 _step‚1061 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚355198427136512 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 845178880 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚262 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚43308610 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 4654602240 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚355198486186496 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 3119740324352 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17104896 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚91083923 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 _runtime‚3437.966990336 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 3113204230756 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 5368709120 KDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚60 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 5494538240 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 122905600 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚310200688865280 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 E8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚91079634 M@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚59049984 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚47775311 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚4287 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚429 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚29744985 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚353108814932580 trainer/global_step‚12899 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚41945088 IBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚73 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚306901692940288 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 4654602240 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚310200759838720 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚20643840 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚349995610701824 PCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚59049984 IDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4876 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚322 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚322 $ _timestamp‚1.786662089432881e+09 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚47775311 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚306901672296448 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚43304323 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚355198427136512 UHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚70973440 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚4287 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4289 D;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚5264 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚10829008 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0¥ÖER2å- _wandbruntime‚3438  _runtime‚3438(çz†æÅM E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚429 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17104896 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚41937920 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 IBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚60 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚352105068199936 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 122905600 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 4552982528 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 3119929447936 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚29747190 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚306924740313088 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 5494538240 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 125829120 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 122905600 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚306924719669248 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 831258624 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 5368709120 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 125829120 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚91086474 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 _runtime‚3438.143579201 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 4654602240 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚322 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚47778866 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚209 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 5368709120 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 3119971396096 NGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚62 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521206 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 4517085184 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚91090769 H@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚665 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚43307610 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚41948160 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚10829851 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 845178880 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 122306526 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚355224980542976 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 3299267883008 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 3113434803368 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 125829120 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚4293 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17104896 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚10829789 D=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚60 G@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚64 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚223 NADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚59053056 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 3119929447936 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚355225039596032 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 5494538240 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚91086474 I@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚4876 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 SFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚62581760 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚353135289686156 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚41348068 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 5368709120 % _timestamp‚1.7866620896131501e+09 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚91090769 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚352105051095040 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚352105051095040 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚47778864 aODeviceStatsMonitor.§sÖuEon_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 3299309820928 _step‚1062 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚353135231298728 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚322 EDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 trainer/global_step‚12899 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 5494538240 >5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5264 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚47778866 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚43311903 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚350021796495360 K>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59053056 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚262 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654602240 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚4295 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚43311903 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚352105068199936 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚40576979ò%P*Ð&çÊM BDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 trainer/global_step‚12899 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 5494538240 >5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5264 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚47778866 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚43311903 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚350021796495360 K>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59053056 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚262 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654602240 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚4295 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚43311903 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚352105068199936 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚40576979 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚429 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17104896 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚41937920 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 IBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚60 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚352105068199936 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 122905600 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 4552982528 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 3119929447936 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚29747190 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚306924740313088 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 5494538240 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 125829120 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 122905600 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚306924719669248 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 831258624 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 5368709120 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 125829120 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚91086474 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 _runtime‚3438.143579201 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 4654602240 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚322 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚47778866 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚209 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 5368709120 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 3119971396096 NGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚62 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521206 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 4517085184 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚91090769 H@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚665 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚43307610 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚41948160 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚10829851 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 845178880 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 122306526 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚355224980542976 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 3299267883008 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 3113434803368 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 125829120 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚4293 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17104896 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚10829789 D=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚60 G@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚64 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚223 NADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚59053056 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 3119929447936 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚355225039596032 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 5494538240 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚91086474 I@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚4876 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 SFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚62581760 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚353135289686156 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚41348068 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 5368709120 % _timestamp‚1.7866620896131501e+09 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚91090769 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚352105051095040 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚352105051095040 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚47778864 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 3299309820928 _step‚1062 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚353135231298728 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚322 EDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚356526130791936 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 4654602240 PCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚59049984 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚47952963 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚223 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚308054040936448 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚322 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚47952961 G>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4876 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚354429666419988 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 4628521206 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚262 trainer/global_step‚12949 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 3131285145088 IDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚209 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚41945088 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 5494538240 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 5368709120 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚353394862751744 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚10869555 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 5368709120 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 125829120 F?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚60 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 122905600 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚356526130791936 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚43469425 UHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚70973440 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 5494538240 @7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚5264 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 4552982528 M@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚59049984 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 3131285145088 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 3311288053760 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 % _timestamp‚1.7866621019792783e+09 E8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚91422386 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚322 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚354429724807368 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚351304900378624 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚4287 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚356526189841920 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚311365328990208 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚322 C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚4289 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚43473712 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17104896 JBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚665 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 125829120 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 4517085184 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 5368709120 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 3131327090176 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 3131327090176 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 122905600 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚43473712 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚429 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚29855237 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 3311338383360 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 845178880 KDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚60 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 _step‚1065 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 122306526 IDeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4289 IBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚75 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚311365399963648 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 _runtime‚3450.512868525 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚353394845646848 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 4654602240 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚41945088 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚40724865 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚351304917417984 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚20643840 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 831258624 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 PCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚58387380 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 125829120 KBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚4876 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚40724790 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚29855235 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚10869628 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚61478912 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚50329600 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚262 IBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚60 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚4287 PIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚73 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚43469425 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚353394862751744 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚353394845646848 BDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 3124807389384 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚47952961 D;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚5264 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚356526189841920 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17104896 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚91426675 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚322ݦ…wòÀO PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚47952963 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 4552982528 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4289 IBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚75 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚311365399963648 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 _runtime‚3450.512868525 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚353394845646848 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 4654602240 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚41945088 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚40724865 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚351304917417984 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚20643840 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 831258624 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 PCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚58387380 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 125829120 KBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚4876 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚40724790 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚29855235 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚10869628 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚61478912 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚50329600 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚262 IBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚60 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚4287 PIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚73 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚43469425 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚353394862751744 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚353394845646848 BDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 3124807389384 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚47952961 D;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚5264 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚356526189841920 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17104896 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚91426675 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚322 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚356526130791936 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 4654602240 PCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚59049984 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚47952963 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚223 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚308054040936448 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚322 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚47952961 G>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4876 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚354429666419988 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 4628521206 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚262 trainer/global_step‚12949 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 3131285145088 IDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚209 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚41945088 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 5494538240 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 5368709120 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚353394862751744 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚10869555 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 5368709120 WIDeviceStatsMonitor.on_train_batch_start/reserve+PMaOd_bytes.small_pool.current‚ 125829120 F?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚60 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 122905600 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚356526130791936 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚43469425 UHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚70973440 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 5494538240 @7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚5264 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 4552982528 M@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚59049984 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 3131285145088 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 3311288053760 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 % _timestamp‚1.7866621019792783e+09 E8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚91422386 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚322 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚354429724807368 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚351304900378624 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚4287 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚356526189841920 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚311365328990208 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚322 C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚4289 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚43473712 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17104896 JBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚665 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 125829120 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 4517085184 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 5368709120 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 3131327090176 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 3131327090176 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 122905600 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚43473712 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚429 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚29855237 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 3311338383360 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 845178880 KDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚60 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 _step‚1065 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 122306526 I6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚322 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚322 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚262 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 4654602240 I@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚4876 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 G@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚60 NADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚58387428 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚91433521 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654602240 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚40727837 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚209 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚308077108953088 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚354456082786136 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚322 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚311388622606336 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚43477005 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚4293 B9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚5264 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 845178880 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚351331086172160 _runtime‚3450.673263492 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚91429226 SFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚70970368 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚223 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 G@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚84 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 BDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 >5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5264 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 125829120 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚353421185089536 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 122905600 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 5368709120 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚91429226 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚354456141173564 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521206 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 5494538240 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚29857442 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 3125037962044 % _timestamp‚1.7866621021429758e+09 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚356552743251456 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 EDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚40727753 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚10870313 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 5368709120 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚322 K>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59053056 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚43472712 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚91433521 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚10870395 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 125829120 IBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚60 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 _step‚1066 PDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚262 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚262 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17104896 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚356552743251456 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 3311584623616 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 122306526 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 5368709120 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚353421167984640 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 3311534297088 D=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚60 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚43477005 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚43472712 NGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚82 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 125829120ì.‰ÉÐ&õÊM NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚47956516 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 122905600 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17104896 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚29857440 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚308077088309248 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 NADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚59053056 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚353421167984640 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 4552982528 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚47956514 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚353421185089536 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 4552982528 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 3131516213760 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 3131558161920 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚41348068 trainer/global_step‚12949 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚47956514 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 5494538240 D6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚322 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚322 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚262 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 4654602240 I@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚4876 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 G@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚60 NADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚58387428 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚91433521 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654602240 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚40727837 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚209 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚308077108953088 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚354456082786136 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚322 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚311388622606336 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚43477005 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚4293 B9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚5264 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 845178880 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚351331086172160 _runtime‚3450.673263492 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚91429226 SFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚70970368 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚223 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 G@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚84 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 BDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 >5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5264 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 125829120 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚353421185089536 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 122905600 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 5368709120 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚91429226 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚354456141173564 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521206 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 5494538240 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚29857442 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 3125037962044 % _timestamp‚1.7866621021429758e+09 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚356552743251456 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 EDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚40727753 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚10870313 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 5368709120 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚322 K>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59053056 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚43472712 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚91433521 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚10870395 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 125829120 IBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚60 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 _step‚1066 PDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚262 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚262 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17104896 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚356552743251456 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 3311584623616 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 122306526 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 5368709120 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚353421167984640 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 3311534297088 D=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚60 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚43477005 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚43472712 NGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚82 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 125829120 EDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚357853834447360 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚44038144 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚29965487 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚357853893497344 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚43634527 G>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4876 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17104896 PIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚69 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚4287 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 3142913856000 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚10910036 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 5368709120 % _timestamp‚1.7866621147786853e+09 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 3142913856000 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚43638814 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚322 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚262 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚262 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 5494538240 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 125829120 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚322 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 4654602240 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚40875452 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚10909967 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 831258624 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 5368709120 trainer/global_step‚12999 _step‚1069 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 125829120 E8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚91765138 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 122306526 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚262 M@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚59049984 C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚4289 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚48130613 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚209 BDeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4289 KDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚60 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 4654602240 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 845178880 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚309206430220288 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 5494538240 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚354710962536448 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚91769427 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 IDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 122905600 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚354710979641344 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚312529983119872 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 3136327851972 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 3142871910912 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚20643840 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 125829120Œ†ûÄ'þ¾O L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 4654602240 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚40875452 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚10909967 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 831258624 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 5368709120 trainer/global_step‚12999 _step‚1069 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 125829120 E8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚91765138 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 122306526 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚262 M@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚59049984 C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚4289 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚48130613 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚209 BDeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4289 KDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚60 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 4654602240 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 845178880 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚309206430220288 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 5494538240 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚354710962536448 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚91769427 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 IDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 122905600 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚354710979641344 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚312529983119872 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 3136327851972 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 3142871910912 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚20643840 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 125829120 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚354710962536448 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 4552982528 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚41945088 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 3136369199992 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚322 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17104896 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚355750576294776 IBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚71 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚29965485 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚312530047801856 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 4628521206 @7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚5264 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚354710979641344 IDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚357853834447360 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚44038144 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚29965487 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚357853893497344 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚43634527 G>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4876 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17104896 PIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚69 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚4287 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 3142913856000 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚10910036 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 5368709120 % _timestamp‚1.7866621147786853e+09 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 3142913856000 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚43638814 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚322 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚262 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚262 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 5494538240 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 125829120 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚322­”(û2ÿ- _wandbruntime‚3463  _runtime‚3463fõÍË&€ÅM NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚43642107 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 3143102979584 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚354737301979136 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚4293 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚309229477593088 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚322 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 3136599772652 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17104896 D8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚262 D=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚60 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚48134166 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚48134164 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 3143102979584 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚48134166 I@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚4876 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 3143144927744 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚48134164 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 3323864182272 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚91776273 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚4295 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚41948160 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 122905600 % _timestamp‚1.7866621149444866e+09 EDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521206 K>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59053056 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17104896 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 122306526 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 5368709120 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 125829120 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚322 _step‚1070 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 122905600 SFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚66776064 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚46132224 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 5368709120 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 5368709120 IBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚60 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 5494538240 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚322 G@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚67 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 831258624 trainer/global_step‚12999 H@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚665 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 845178880 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚352640392888320 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚10910792 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚309229456949248 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 5494538240 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚61478912 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚43637814 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚357880446906880 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚91776273 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚41348068 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚29967690 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚20643840 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚355776992660972 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 125829120 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚262 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚29967692 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚352640375848960 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 4552982528 E6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚322 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 125829120 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚357880446906880 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚43642107 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚40878482 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚4293 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 4654602240 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚354737301979136 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 4517085184 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 3143144927744 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚91771978 PDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654602240 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚43637814 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚40878549 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚312553274999296 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚355776934273544 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 4552982528 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 B9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚5264 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚262 G@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚60 >5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5264 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚357880387853824 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 3323818050048 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 _runtime‚3463.474917966(ÍIÍ&ÊM OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚40878549 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚312553274999296 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚355776934273544 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 4552982528 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 B9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚5264 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚262 G@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚60 >5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5264 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚357880387853824 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 3323818050048 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 _runtime‚3463.474917966 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚43642107 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 3143102979584 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚354737301979136 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚4293 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚309229477593088 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚322 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 3136599772652 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17104896 D8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚262 D=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚60 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚48134166 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚48134164 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 3143102979584 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚48134166 I@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚4876 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 3143144927744 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚48134164 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 3323864182272 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚91776273 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚4295 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚41948160 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 122905600 % _timestamp‚1.7866621149444866e+09 EDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521206 K>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59053056 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17104896 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 122306526 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 5368709120 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 125829120 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚322 _step‚1070 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 122905600 SFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚66776064 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚46132224 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 5368709120 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 5368709120 IBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚60 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 5494538240 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚322 G@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚67 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 831258624 trainer/global_step‚12999 H@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚665 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 845178880 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚352640392888320 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚10910792 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚309229456949248 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 5494538240 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚61478912 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚43637814 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚357880446906880 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚91776273 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚41348068 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚29967690 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚20643840 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚355776992660972 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 125829120 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚262 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚29967692 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚352640375848960 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 4552982528 E6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚322 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 125829120 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚357880446906880 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚43642107 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚40878482 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚4293 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 4654602240 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚354737301979136 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 4517085184 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 3143144927744 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚91771978 PDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654602240 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚43637814¦jž®;Å2‚- _wandbruntime‚3463  _runtime‚3463Wdó´sƒí 0loss/FRt2/assignment_loss‚0.4618221340245352 /loss/FRt2/detection_loss‚0.4632103126830147 %loss/total_loss‚2.053133735787165 _runtime‚3463.475360964 _step‚1071 0loss/FRt1/assignment_loss‚0.5916204276919269 .loss/FRt1/detection_loss‚0.536480861387688 epoch‚32 trainer/global_step‚12999 % _timestamp‚1.7866621149454806e+09xJ“x„ò 0loss/FRt2/assignment_loss‚0.4618221340245352 /loss/FRt2/detection_loss‚0.4632103126830147 %loss/total_loss‚2.053133735787165 _runtime‚3463.475360964 _step‚1071 0loss/FRt1/assignment_loss‚0.5916204276919269 .loss/FRt1/detection_loss‚0.536480861387688 epoch‚32 trainer/global_step‚12999 % _timestamp‚1.7866621149454806e+09¯ØÑ[ú …‚(:• ä™ùÓÍ–‰á gpu.0.gpu‚54.0 gpu.0.memory‚9, gpu.0.memoryAllocated‚7.0313262939453125) gpu.0.memoryAllocatedBytes‚ 6039863296 gpu.0.temp‚36.0 gpu.0.powerWatts‚121.728' gpu.0.enforcedPowerLimitWatts‚500.0) gpu.0.powerPercent‚24.345599999999997 gpu.0.smClock‚1410 gpu.0.memoryClock‚1593$ gpu.0.correctedMemoryErrors‚1955# gpu.0.uncorrectedMemoryErrors‚0ß5·ž›†‚(: ä™ùÓŒ‹¬ã disk.sda3.in‚31646.87890625( disk.nvme0c0n1.out‚58803.72705078125 disk.sda.in‚31646.87890625" disk.nvme0n1.in‚43794.48046875 disk.dm-1.in‚31646.87890625# disk.dm-0.out‚58803.72705078125# memory_percent‚68.6166763305664$ disk./.usageGB‚487.3773078918457$ disk.nvme0c0n1.in‚43794.48046875 disk.sda2.in‚0 disk.sda2.out‚0 disk.dm-1.out‚ 63.89453125 disk.dm-0.in‚43794.48046875# proc.memory.rssMB‚ 2362.08984375* proc.memory.percent‚14.417052268981934 disk.sda3.out‚ 63.89453125 disk.sda.out‚ 63.89453125 disk.sda1.out‚0& disk.nvme0n1.out‚58803.72705078125 disk.drbd2165.out‚0 cpu‚27.20657163041467 proc.cpu.threads‚17 network.sent‚ 1.68212e+07) disk./.usagePercent‚32.70830006942106 disk.sda1.in‚0 disk.drbd2165.in‚0& proc.memory.availableMB‚ 5141.84375 network.recv‚ 6.40652e+06˜"ñ2‡- _wandbruntime‚3475  _runtime‚3475ž—kƒÈˆÂ _step‚1072 & lr-AdamW/pg1‚0.0004192901420148928 & lr-AdamW/pg2‚0.0004192901420148928 trainer/global_step‚13049 $ _timestamp‚1.786662127401406e+09 _runtime‚3475.931421211åNïÎÍ‰Ç _runtime‚3475.931421211 _step‚1072 & lr-AdamW/pg1‚0.0004192901420148928 & lr-AdamW/pg2‚0.0004192901420148928 trainer/global_step‚13049 $ _timestamp‚1.786662127401406e+09°èn2Š- _wandbruntime‚3475  _runtime‚3475ýW[OÁ'‹»O PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚43803916 UHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚70973440 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚20643840 IDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4876 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 3154458676736 IBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚75 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 125829120 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚359181538102784 PIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚73 PCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚58387380 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚357071369394804 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 3147889662580 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 122306526 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 5368709120 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚262 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚48308261 F>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 4552982528 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚43799629 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚429 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚262 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚313694599331840 @7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚5264 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17104896 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚356027079426048 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 5368709120 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 125829120 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚209 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 % _timestamp‚1.7866621274023516e+09 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 4654602240 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚41026580 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚50329600 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 3147931010600 M@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚59049984 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 4552982528 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 5494538240 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 4654602240 IDeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4289 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚10950918 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 845178880 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚322 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 125829120 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚322 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚262 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚92112179 KBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚4876 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 4517085184 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚43799629 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚359181597152768 IBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚60 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 trainer/global_step‚13049 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 122905600 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 3154500621824 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 5368709120 _runtime‚3475.935896894 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 3335821115392 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0VaâÆ'ŒÀO ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 3147889662580 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 122306526 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 5368709120 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚262 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚48308261 F>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 4552982528 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚43799629 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚429 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚262 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚313694599331840 @7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚5264 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17104896 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚356027079426048 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 5368709120 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 125829120 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚209 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 % _timestamp‚1.7866621274023516e+09 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 4654602240 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚41026580 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚50329600 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 3147931010600 M@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚59049984 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 4552982528 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 5494538240 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 4654602240 IDeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4289 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚10950918 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 845178880 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚322 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 125829120 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚322 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚262 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚92112179 KBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚4876 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 4517085184 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚43799629 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚359181597152768 IBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚60 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 trainer/global_step‚13049 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 122905600 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 3154500621824 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 5368709120 _runtime‚3475.935896894 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 3335821115392 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚43803916 UHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚70973440 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚20643840 IDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4876 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 3154458676736 IBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚75 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 125829120 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚359181538102784 PIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚73 PCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚58387380 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚357071369394804±‘5¤£2- _wandbruntime‚3476  _runtime‚3476–ø#-‘%ŽÃM XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 831258624 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 5368709120 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚48311816 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 3336116636672 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚359208150562304 G@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚60 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚92119025 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 5368709120 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 125829120 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚48311814 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚41029583 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚353949682565120 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚322 _runtime‚ 3476.10108175 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 3154689745408 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 122905600 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 5494538240 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 5368709120 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 4552982528 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚41029667 IBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚60 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚48311816 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654602240 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚356053401763840 EDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 >5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5264 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚4293 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 3336066310144 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚357097785760952 NGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚82 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚41348068 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚322 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚262 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17104896 NADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚59053056 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚223 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚61478912 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521206 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚359208091509248 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚48311814 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚43807209 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚43807209 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚356053418868736 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚92114730 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚30077940 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 3148120235192 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 125829120 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚4295 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚359208150562304 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚10951643 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚310381846233088 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚50326528 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 5494538240 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚43802916 I@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚4876 G@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚84 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 B6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚322 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚92119025 H@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚665 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚43802916 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚10951725 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚313717891899392 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 122306526 D=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚60 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 4552982528 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚41948160 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 trainer/global_step‚13049 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 4654602240 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 K>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59053056 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚429 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚30077942 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚209 SFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚70970368 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚310381825589248 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚353949665525760 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚262 B9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚5264 _step‚1074 P6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚322 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚92119025 H@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚665 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚43802916 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚10951725 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚313717891899392 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 122306526 D=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚60 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 4552982528 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚41948160 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 trainer/global_step‚13049 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 4654602240 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 K>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59053056 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚429 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚30077942 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚209 SFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚70970368 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚310381825589248 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚353949665525760 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚262 B9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚5264 _step‚1074 PDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 5368709120 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 125829120 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚48311814 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚41029583 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚353949682565120 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚322 _runtime‚ 3476.10108175 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 3154689745408 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 122905600 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 5494538240 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 5368709120 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 4552982528 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚41029667 IBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚60 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚48311816 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654602240 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚356053401763840 EDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 >5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5264 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚4293 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 3336066310144 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚357097785760952 NGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚82 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚41348068 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚322 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚262 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17104896 NADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚59053056 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚223 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚61478912 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521206 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚359208091509248 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚48311814²nWE2- _wandbruntime‚3476  _runtime‚3476ëòæv‘ð 0loss/FRt1/assignment_loss‚0.5340467760361968 trainer/global_step‚13049 _step‚1075 1loss/FRt2/assignment_loss‚0.45952815674072384 /loss/FRt1/detection_loss‚0.4570443387571966 /loss/FRt2/detection_loss‚0.5012473609597162 &loss/total_loss‚1.9518666324938334 epoch‚32 % _timestamp‚1.7866621275718153e+09 _runtime‚3476.101654793”7•º{’õ /loss/FRt2/detection_loss‚0.5012473609597162 &loss/total_loss‚1.9518666324938334 epoch‚32 % _timestamp‚1.7866621275718153e+09 _runtime‚3476.101654793 0loss/FRt1/assignment_loss‚0.5340467760361968 trainer/global_step‚13049 _step‚1075 1loss/FRt2/assignment_loss‚0.45952815674072384 /loss/FRt1/detection_loss‚0.4570443387571966³1B‚«¡“‚(:– ó™ùÓ–õìà gpu.0.gpu‚58.0 gpu.0.memory‚10, gpu.0.memoryAllocated‚7.0313262939453125) gpu.0.memoryAllocatedBytes‚ 6039863296 gpu.0.temp‚35.0 gpu.0.powerWatts‚124.524' gpu.0.enforcedPowerLimitWatts‚500.0) gpu.0.powerPercent‚24.904799999999998 gpu.0.smClock‚1410 gpu.0.memoryClock‚1593$ gpu.0.correctedMemoryErrors‚1955# gpu.0.uncorrectedMemoryErrors‚0vÆÈ”ž”‚(:“ ó™ùÓõ›Ôã network.sent‚ 1.6907336e+07 network.recv‚ 6.429395e+06$ disk./.usageGB‚487.3771324157715 disk.dm-1.in‚ 32534.8515625 disk.dm-1.out‚ 64.16796875 disk.drbd2165.in‚0 disk.sda3.in‚ 32534.8515625 proc.cpu.threads‚17 disk.sda2.in‚0" disk.nvme0n1.in‚43803.81640625% disk.nvme0n1.out‚58807.5322265625 disk.drbd2165.out‚0 disk.dm-0.in‚43803.81640625$ disk.nvme0c0n1.in‚43803.81640625' disk.nvme0c0n1.out‚58807.5322265625 disk.sda.out‚ 64.16796875* disk./.usagePercent‚32.708288293074474 disk.sda2.out‚0 disk.sda.in‚ 32534.8515625# proc.memory.rssMB‚ 2362.08984375* proc.memory.percent‚14.417052268981934 cpu‚27.206467553410707 disk.sda1.in‚0 disk.sda1.out‚0" disk.dm-0.out‚58807.5322265625 disk.sda3.out‚ 64.16796875$ memory_percent‚68.62046718597412) proc.memory.availableMB‚ 5141.22265625sq1¯2•- _wandbruntime‚3488  _runtime‚3488“³=œË–Å _runtime‚3488.609464452 _step‚1076 ' lr-AdamW/pg1‚0.00041390601598882915 ' lr-AdamW/pg2‚0.00041390601598882915 trainer/global_step‚13099 % _timestamp‚1.7866621400794415e+09¥¿[Ð—Ê trainer/global_step‚13099 % _timestamp‚1.7866621400794415e+09 _runtime‚3488.609464452 _step‚1076 ' lr-AdamW/pg1‚0.00041390601598882915 ' lr-AdamW/pg2‚0.00041390601598882915´ñ…}Ñ2˜- _wandbruntime‚3488  _runtime‚3488RiZÁ'™»O E8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚92450642 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 4552982528 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4289 IDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 _step‚1077 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚357343196315648 JBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚665 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚262 F?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚60 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17104896 KBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚4876 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚360509300808192 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 4654602240 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚48485913 IDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 KDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚60 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 4654602240 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 3166045442560 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 3348100703232 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚61478912 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚360509300808192 PCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚59049984 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚20643840 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 3348148935680 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 5494538240 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚48485913 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚357343213420544 % _timestamp‚1.7866621400803723e+09 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚48485911 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚223 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 3159451473188 C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚4289 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚4287 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚360509241758208 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚360509241758208 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚314859316435968 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 831258624 BDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4876 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 122306526 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚43969018 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚41177587 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 5494538240 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚322 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚30185985 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚262 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚357343213420544 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 5368709120 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 3166087387648 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚43969018 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚311511167500288 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚41348020 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚322 IBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚60 _runtime‚3488.613870654 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚41945088 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 125829120 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 UHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚68876288 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0k–‘}%šÀO PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚43969018 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚311511167500288 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚41348020 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚322 IBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚60 _runtime‚3488.613870654 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚41945088 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 125829120 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 UHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚68876288 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 E8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚92450642 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 4552982528 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4289 IDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 _step‚1077 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚357343196315648 JBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚665 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚262 F?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚60 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17104896 KBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚4876 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚360509300808192 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 4654602240 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚48485913 IDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 KDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚60 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 4654602240 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 3166045442560 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 3348100703232 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚61478912 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚360509300808192 PCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚59049984 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚20643840 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 3348148935680 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 5494538240 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚48485913 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚357343213420544 % _timestamp‚1.7866621400803723e+09 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚48485911 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚223 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 3159451473188 C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚4289 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚4287 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚360509241758208 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚360509241758208 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚314859316435968 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 831258624 BDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4876 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 122306526 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚43969018 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚41177587 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 5494VàI538240 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚322 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚30185985 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚262 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚357343213420544 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 5368709120 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 3166087387648µ…òä›2›- _wandbruntime‚3488  _runtime‚3488ñaK~É&œÃM ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚41180614 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654602240 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 5494538240 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚322 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 _step‚1078 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚429 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚322 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚360535795164672 G@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚85 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚322 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚43972311 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17104896 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚41348068 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 4552982528 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 4552982528 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚314882539242496 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚262 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚92457482 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 4654602240 E8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 # _timestamp‚1.78666214024611e+09 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚43972311 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚209 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 5368709120 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚322 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚30188192 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚20643840 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚52423680 PDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚92461777 EDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59053056 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17104896 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 3166318459392 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚360535854217728 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 3166318459392 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚311534194229248 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚43968018 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 3166276511232 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚41948160 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 122905600 NGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚83 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 3166276511232 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521206 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 3159682045800 DDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚48489466 >5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5264 NADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚59053056 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚314882612310016 NADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚58387428 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 125829120 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0ù}³Î&ÈM H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚4293 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚357369535758336 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 5368709120 B9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚5264 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚4293 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚48489464 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚358418637248360 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚262 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 5494538240 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚262 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚357369518653440 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚61478912 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚41180699 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚10992507 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚311534214873088 G@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚60 D=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚60 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 125829120 IBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚60 H@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚665 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 3348397436928 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 BDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59053056 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17104896 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 3166318459392 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚360535854217728 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 3166318459392 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚311534194229248 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚43968018 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 3166276511232 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚41948160 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 122905600 NGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚83 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 3166276511232 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521206 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 3159682045800 DDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚48489466 >5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5264 NADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚59053056 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚314882612310016 NADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚58387428 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 125829120 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚41180614 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654602240 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 5494538240 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚322 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 _step‚1078 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚429 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚322 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚360535795164672 G@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚85 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚322 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚43972311 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17104896 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚41348068 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 4552982528 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 4552982528 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚314882539242496 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚262 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚92457482 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 4654602240 E8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 # _timestamp‚1.78666214024611e+09 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚43972311 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚209 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 5368709120 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚322 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚30188192 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚20643840 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚52423680 PDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚92461777 EDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4876 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17104896 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚4287 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚358659330310144 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚429 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 4628521206 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚41348020 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 KDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚60 F>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 3177674153472 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚41945088 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 122905600 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4289 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 845178880 @7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚5264 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 5368709120 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚262 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚44129833 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚223 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚312663536140288 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 IBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚60 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 3177632208384 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚48663561 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚44129833 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 BDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚361836945413632 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17104896 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚312663515496448 M@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚59049984 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 3177632208384 IBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚74 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚30296235 PIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚72 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚356542059085824 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚322 PCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚59049984 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 4654602240 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚20643840 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚41328494 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚61478912 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 4517085184 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 5494538240 D;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚5264 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚41328568 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚316023910393344 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 5368709120 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 4552982528 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚41945088 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚356542076125184 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚262 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚48663561 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚361837004463616 UHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚68876288 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 trainer/global_step‚13149 % _timestamp‚1.7866621526791801e+09 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 3177674153472 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚358659330310144 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚30296237 \ODeviceStatsMonitor.on_trai”ªj>ˆn_batch_start/inactive_split_bytes.small_pool.current‚48232448 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 125829120 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚322 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚358659313205248 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚11032259 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 831258624 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚359713072369620sÛ¹Æ'¨ÀO M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚41328494 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚61478912 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 4517085184 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 5494538240 D;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚5264 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚41328568 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚316023910393344 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 5368709120 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 4552982528 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚41945088 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚356542076125184 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚262 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚48663561 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚361837004463616 UHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚68876288 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 trainer/global_step‚13149 % _timestamp‚1.7866621526791801e+09 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 3177674153472 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚358659330310144 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚30296237 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚48232448 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 125829120 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚322 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚358659313205248 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚11032259 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 831258624 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚359713072369620 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 3171054631816 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 5494538240 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 3360443129344 PCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚58387380 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚44134120 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 5368709120 JBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚665 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚262 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 125829120 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚44134120 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚316023979269632 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 3360394896896 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 E8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚92793394 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚361836945413632 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚359713130757000 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 125829120 _runtime‚3501.209464756 G>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4876 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17104896 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚4287 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚358659330310144 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚429 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 4628521206 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚41348020 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 KDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚60 F>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 3177674153472 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚41945088 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 122905600 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4289 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 845178880 @7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚5264 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 5368709120 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚262 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚44129833 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚223 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚312663536140288 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 IBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚60 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 3177632208384 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚48663561 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚44129833 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 BDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚361836945413632 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17104896 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚312663515496448 M@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚59049984 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 3177632208384 IBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚74 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚30296235 PIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚72 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚356542059085824 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚322 PCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚59049984 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 4654602240 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚20643840 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0¹ý5V2©- _wandbruntime‚3501  _runtime‚3501ršÒÐË&ªÅM QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 4552982528 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚44133120 NGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚74 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚262 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 5368709120 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚48667116 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 4654602240 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 122905600 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚223 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚11033070 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚48229376 IBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚60 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚262 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚92804529 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚262 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚44137413 B9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚5264 G@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚76 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚48667114 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚209 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚316047202797056 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚312686583513088 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 5368709120 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚41948160 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚48667116 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 3360688157184 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚359739547123196 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚322 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚361863557873152 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚30298440 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 4517085184 >5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5264 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚48667114 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 5494538240 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚4293 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚358685652647936 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 3177863277056 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚312686562869248 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 5368709120 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚20643840 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 831258624 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚322 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚359739488735768 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚322 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚429 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚92800234 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚11033144 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 3171285204476 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚358685635543040 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 4552982528 EDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚41331510 NADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚58387428 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚356568261918720 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚92800234 NADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚59053056 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚4295 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚44133120 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17104896 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚358685652647936 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521206 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 122905600 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654602240 H@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚665 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚44137413 EDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59053056 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚92804529 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚356568244879360 trainer/global_step‚13149 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚316047271670272 D=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚60 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 125829120 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚41948160 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚4293 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 3360639927808 G@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚60 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚30298442 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 % _timestamp‚1.7866621528458593e+09ô¶wmÐ&«ÊM E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚92800234 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚11033144 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 3171285204476 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚358685635543040 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 4552982528 EDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚41331510 NADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚58387428 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚356568261918720 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚92800234 NADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚59053056 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚4295 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚44133120 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17104896 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚358685652647936 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521206 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 122905600 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654602240 H@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚665 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚44137413 EDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59053056 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚92804529 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚356568244879360 trainer/global_step‚13149 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚316047271670272 D=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚60 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 125829120 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚41948160 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚4293 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 3360639927808 G@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚60 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚30298442 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 % _timestamp‚1.7866621528458593e+09 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 4552982528 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚44133120 NGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚74 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚262 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 5368709120 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚48667116 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 4654602240 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 122905600 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚223 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚11033070 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚48229376 IBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚60 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚262 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚92804529 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚262 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚44137413 B9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚5264 G@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚76 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚48667114 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚209 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚316047202797056 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚312686583513088 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 5368709120 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚41948160 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚48667116 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 3360688157184 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚359739547123196 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚322 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚361863557873152 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚30298440 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 4517085184 >5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5264 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚48667114 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 5494538240 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚4293 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚358685652647936 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 3177863277056 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚312686562869248 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 5368709120 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚20643840 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 831258624 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚322 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚359739488735768 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚322 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚429 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2ºa‰ž‰2¬- _wandbruntime‚3501  _runtime‚3501@eƒt­î .loss/FRt1/detection_loss‚0.475262697127939 /loss/FRt2/detection_loss‚0.5205692217403588 %loss/total_loss‚2.016307015261004 epoch‚32 0loss/FRt1/assignment_loss‚0.5318084894549037 1loss/FRt2/assignment_loss‚0.48866660693780223 trainer/global_step‚13149 % _timestamp‚1.7866621528469481e+09 _runtime‚3501.376804665 _step‚1083± Sy®ó 0loss/FRt1/assignment_loss‚0.5318084894549037 1loss/FRt2/assignment_loss‚0.48866660693780223 trainer/global_step‚13149 % _timestamp‚1.7866621528469481e+09 _runtime‚3501.376804665 _step‚1083 .loss/FRt1/detection_loss‚0.475262697127939 /loss/FRt2/detection_loss‚0.5205692217403588 %loss/total_loss‚2.016307015261004 epoch‚32»€Üì–¯‚(:‹ ‘šùÓ‡‰ä gpu.0.gpu‚64.0 gpu.0.memory‚12, gpu.0.memoryAllocated‚7.0313262939453125) gpu.0.memoryAllocatedBytes‚ 6039863296 gpu.0.temp‚36.0 gpu.0.powerWatts‚209.133' gpu.0.enforcedPowerLimitWatts‚500.0 gpu.0.powerPercent‚41.8266 gpu.0.smClock‚1410 gpu.0.memoryClock‚1593$ gpu.0.correctedMemoryErrors‚1955# gpu.0.uncorrectedMemoryErrors‚0Ó÷ Õê°‚(: ‘šùÓÎÁ±æ disk.drbd2165.in‚0 disk.dm-0.in‚43803.81640625# disk.dm-0.out‚58857.58544921875 disk.sda3.in‚ 32534.8515625 disk.sda3.out‚64.5( proc.memory.availableMB‚ 5141.4453125 network.sent‚ 1.7052162e+07 disk.sda1.out‚0 disk.dm-1.in‚ 32534.8515625( disk.nvme0c0n1.out‚58857.58544921875* proc.memory.percent‚14.417052268981934 cpu‚27.205805078408805 network.recv‚ 6.488034e+06 disk.sda1.in‚0 disk.sda2.out‚0" disk.nvme0n1.in‚43803.81640625& disk.nvme0n1.out‚58857.58544921875 disk.dm-1.out‚64.5$ disk.nvme0c0n1.in‚43803.81640625# proc.memory.rssMB‚ 2362.08984375% disk./.usageGB‚487.49489974975586 disk.sda2.in‚0 disk.sda.in‚ 32534.YoÈ¢8515625 disk.sda.out‚64.5$ memory_percent‚68.61910820007324 proc.cpu.threads‚17 disk.drbd2165.out‚0* disk./.usagePercent‚32.716191757671545Ñ7L²2±- _wandbruntime‚3514  _runtime‚3514I±H(Ê²Ä ' lr-AdamW/pg1‚0.00040320286842823267 ' lr-AdamW/pg2‚0.00040320286842823267 trainer/global_step‚13199 $ _timestamp‚1.786662165845488e+09 _runtime‚3514.375492486 _step‚1084Ø×âÏ³É $ _timestamp‚1.786662165845488e+09 _runtime‚3514.375492486 _step‚1084 ' lr-AdamW/pg1‚0.00040320286842823267 ' lr-AdamW/pg2‚0.00040320286842823267 trainer/global_step‚13199¼M®çm2´- _wandbruntime‚3514  _runtime‚3514úÈØ­Á'µ»O TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17104896 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚44299222 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚50329600 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚359975447199744 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 3189218974208 JBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚665 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚20643840 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚363164708119040 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚359975430094848 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 122306526 IBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚60 IDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚361033923857028 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚48841213 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 845178880 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚262 _step‚1085 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 4654602240 PCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚59049984 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚93140435 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚429 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚30406485 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚262 trainer/global_step‚13199 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 5494538240 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 125829120 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 4552982528 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚48841213 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚357851365801984 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚4287 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚44294935 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚41479574 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 125829120 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 4552982528 KBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚4876 PCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚58387380 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 5494538240 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 KDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚60 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚48841211 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚44299222 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚359975447199744 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚313815884136448 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚322 _runtime‚3514.379864799 M@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚59049984 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚322 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚209 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚317188575549440 F?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚60 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4289 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚44294935 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚317188646522880 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚322 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 G>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4876 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚363164649069056 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚41348020 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 E8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚93136146 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 3189218974208 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 4654602240 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚4287 UHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚70973440 D;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚5264 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚48841211 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚313815904780288 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 3372691412992 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 5494538240 @7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚5264 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17104896 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 122905600 IBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚74 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚11073089 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 831258624 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 4628521206 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 3182616442424 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚363164649069056 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚41479648 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚357851348762624 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 5368709120 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 3182575094404 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 5368709120 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚322 IDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 5368709120 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 3189260919296 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚41945088 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚61478912 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 45170851848FeÆ'¶ÀO PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚48841213 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 845178880 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚262 _step‚1085 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 4654602240 PCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚59049984 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚93140435 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚429 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚30406485 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚262 trainer/global_step‚13199 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 5494538240 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 125829120 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 4552982528 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚48841213 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚357851365801984 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚4287 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚44294935 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚41479574 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 125829120 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 4552982528 KBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚4876 PCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚58387380 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 5494538240 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 KDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚60 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚48841211 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚44299222 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚359975447199744 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚313815884136448 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚322 _runtime‚3514.379864799 M@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚59049984 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚322 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚209 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚317188575549440 F?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚60 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4289 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚44294935 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚317188646522880 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚322 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 G>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4876 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚363164649069056 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚41348020 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 E8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚93136146 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 3189218974208 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 4654602240 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚4287 UHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚70973440 D;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚5264 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚48841211 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚313815904780288 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 3372691412992 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 5494538240 @7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚5264 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17104896 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 122905600 IBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚74 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚11073089 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 831258624 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 4628521206 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 3182616442424 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚363164649069056 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚41479648 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚357851348762624 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 5368709120 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 3182575094404 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 5368709120 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚322 IDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 5368709120 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 3189260919296 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚41945088 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚61478912 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 4517085184 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17104896 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚44299222 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚50329600 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚359975447199744 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 3189218974208 JBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚665 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚20643840 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚363164708119040 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚359975430094848 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 122306526 IBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚60 IDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚361033923857028½9Ù~'2·- _wandbruntime‚3514  _runtime‚3514WÕ!ªË&¸ÅM G@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚60 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 4552982528 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 3189450042880 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚41482574 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 831258624 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚93142986 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚48844764 K>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59053056 P6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚322 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 NGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚70 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚61478912 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 5368709120 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚44298222 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚361060398610604 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 125829120 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚48844766 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 5494538240 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 5368709120 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17104896 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 122905600 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 122905600 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚48844764 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚11073884 IBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚60 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚429 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚262 DDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521206 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 122306526 >5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5264 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚363191261528576 G@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚72 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 _step‚1086 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚41948160 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚44302515 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 3372936443904 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 NADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚58387428 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚262 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚363191261528576 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚360001769537536 I@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚4876 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚11073954 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚48844766 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚317211938923520 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 845178880 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 125829120 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚322 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚4295 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚44302515 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚4293 B9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚5264 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚209 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚361060340223176 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 4654602240 NADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚59053056 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654602240 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚317211867953152 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚363191202475520 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 3189450042880 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚262 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚93147281 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚44298222 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 4552982528 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 3189491991040 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚93142986 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚41482646 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚30408692 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 B8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 D=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚60 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚223 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 3182805667016 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚93147281 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚360001752432640 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚313838931509248 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚322 % _timestamp‚1.7866621660078151e+09 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚4293 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚322 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 125829120 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚313838952153088 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚41348068}ùs—¹ÊM ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚50326528 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚322 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 NGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚70 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚61478912 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 5368709120 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚44298222 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚361060398610604 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 125829120 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚48844766 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 5494538240 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 5368709120 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17104896 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 122905600 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 122905600 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚48844764 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚11073884 IBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚60 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peIH½Ù9 ak‚429 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚262 DDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521206 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 122306526 >5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5264 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚363191261528576 G@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚72 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 _step‚1086 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚41948160 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚44302515 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 3372936443904 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 NADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚58387428 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚262 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚363191261528576 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚360001769537536 I@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚4876 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚11073954 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚48844766 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚317211938923520 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 845178880 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 125829120 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚322 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚4295 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚44302515 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚4293 B9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚5264 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚209 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚361060340223176 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 4654602240 NADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚59053056 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654602240 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚317211867953152 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚363191202475520 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 3189450042880 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚262 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚93147281 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚44298222 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 4552982528 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 3189491991040 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚93142986 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚41482646 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚30408692 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 B8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 D=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚60 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚223 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 3182805667016 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚93147281 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚360001752432640 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚313838931509248 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚322 % _timestamp‚1.7866621660078151e+09 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚4293 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚322 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 125829120 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚313838952153088 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚41348068 G@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚60 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 4552982528 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 3189450042880 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚41482574 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 831258624 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚93142986 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚48844764 K>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59053056 PDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4876 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚364887416713216 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚361682501304320 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 % _timestamp‚1.7866621833869596e+09 @7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚5264 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚361682484199424 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚44482224 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚318656583792128 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚315267543728128 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 5494538240 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 4654602240 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚20643840 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚49079467 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚41945088 PIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚78 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 3389060707840 PCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚58387380 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 4552982528 D;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚5264 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚49079467 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚11121415 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚209 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 IBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚60 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚364887357663232 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚49079469 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚362739799195757 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 3198244225133 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚44486511 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 3204873463808 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚41678764 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚362739740808377 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 F?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚60 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17104896 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚322 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚322 IDeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4289 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚4289 E8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚93561691 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚4287 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚41945088 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 4654602240 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 3204915408896 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚41678844 _step‚1090 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 122306526 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 5368709120 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚93565980 IDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 4552982528§mIP#ÇÀO L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 4654602240 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚20643840 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚49079467 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚41945088 PIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚78 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 3389060707840 PCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚58387380 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 4552982528 D;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚5264 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚49079467 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚11121415 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚209 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 IBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚60 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚364887357663232 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚49079469 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚362739799195757 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 3198244225133 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚44486511 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 3204873463808 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚41678764 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚362739740808377 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 F?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚60 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17104896 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚322 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚322 IDeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4289 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚4289 E8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚93561691 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚4287 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚41945088 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 4654602240 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 3204915408896 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚41678844 _step‚1090 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 122306526 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 5368709120 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚93565980 IDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 4552982528 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 4628521206 M@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚59049984 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 122905600 UHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚73070592 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚61478912 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚322 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚262 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 3204873463808 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 3204915408896 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚30557349 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚223 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 125829120 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 122905600 trainer/global_step‚13249 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17104896 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚429 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚315267523084288 BDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4876 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚364887416713216 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚361682501304320 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 % _timestamp‚1.7866621833869596e+09 @7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚5264 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚361682484199424 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚44482224 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚318656583792128 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚315267543728128 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 5494538240Â27œZ2È- _wandbruntime‚3532  _runtime‚3532ÑgËË&ÉÅM QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 845178880 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚49083022 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚318679878981120 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚322 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 5368709120 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚93572826 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚315290591100928 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚46132224 NADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚58387428 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚49083022 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚41681824 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 3198474797793 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 122306526 IBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚60 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚361708823642112 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚318679945757184 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚61478912 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚41948160 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚11122268 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚322 trainer/global_step‚13249 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚93572826 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚41948160 EDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59053056 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚93568531 >5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5264 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚44489804 P8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 BDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚41681750 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚429 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 3198433449725 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 125829120 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 EDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654602240 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17104896 NGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚72 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521206 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚262 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 831258624 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚322 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 5494538240 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 5368709120 D=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚60 H@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚665 % _timestamp‚1.7866621835493944e+09 B9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚5264 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 3205146480640 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚44489804 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 3389308524032 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 5494538240 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚364913970122752 I@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚4876 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚20643840 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚315290570457088 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 5494538240 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚223 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17104896 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚93568531 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚11122196 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚44485511 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 4654602240 NADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚59053056 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚361708823642112 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 4552982528 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚262 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 122905600 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚362766215561953À †«Ð&ÊÊM YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 3205146480640 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚361708806537216 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚359567723724800 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚4295 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚364913970122752 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚362766157174525 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚4293 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 3389354656256 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚262 G@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚60 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚4293 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚359567740764160 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 4517085184 DDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59053056 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚93568531 >5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5264 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚44489804 P8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 BDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚41681750 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚429 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 3198433449725 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 125829120 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 EDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654602240 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17104896 NGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚72 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521206 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚262 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 831258624 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚322 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 5494538240 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 5368709120 D=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚60 H@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚665 % _timestamp‚1.7866621835493944e+09 B9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚5264 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 3205146480640 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚44489804 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 3389308524032 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 5494538240 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚364913970122752 I@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚4876 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚20643840 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚315290570457088 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 5494538240 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚223 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17104896 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚93568531 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚11122196 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚44485511 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 4654602240 NADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚59053056 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚361708823642112 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 4552982528 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚262 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 122905600 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚362766215561953 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 845178880 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚49083022 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚318679878981120 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚322 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 5368709120 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚93572826 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚315290591100928 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚46132224 NADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚58387428 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚49083022 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚41681824 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 3198474797793 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 122306526 IBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚60 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚361708823642112 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚318679945757184 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚61478912 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚41948160 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚11122268 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚322 trainer/global_step‚13249 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚93572826 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚41948160 EDeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4289 IDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 5494538240 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 5368709120 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚322 C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚4289 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚11161904 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚364060650683165 _step‚1094 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚44651613 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚262 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚366215061318656 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 4628521206 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 4517085184 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 122306526 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 4552982528 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 122905600 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚41829441 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚223 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 3209764687721 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 125829120 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚4287 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 122905600 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚30667601 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 3401372004352 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 G>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4876 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚209 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚41348020 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚44651613 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚316419912368128 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚360850844647424 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 4552982528 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 3401323771904 F?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚60 @7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚5264 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚366215120368640 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚362998601089024 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚366215120368640 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚41945088 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 BDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 831258624 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 5494538240 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 % _timestamp‚1.7866621961982937e+09 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚322 M@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚59049984 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 3216502174720 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚429 PIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚62 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚364060592295785 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚49257119 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚319821284372480 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 4654602240 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 3216502174720 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚49257117 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 4654602240 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚362998601089024 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.curren¦w~D±t‚41945088 D;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚5264 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚319821215496192 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚61478912 _runtime‚3544.731929513 IDeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4289 IDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 5494538240 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 5368709120 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚322 C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚4289 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚11161904 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚364060650683165 _step‚1094 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚44651613 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚262 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚366215061318656 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 4628521206 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 4517085184 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 122306526 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 4552982528 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 122905600 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚41829441 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚223 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 3209764687721 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 125829120 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚4287 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 122905600 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚30667601 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 3401372004352 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 G>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4876 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚209 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚41348020 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚44651613 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚316419912368128 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚360850844647424 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 4552982528 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 3401323771904 F?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚60 @7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚5264 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚366215120368640 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚362998601089024 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚366215120368640 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚41945088 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 BDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 831258624 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 5494538240 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 % _timestamp‚1.7866621961982937e+09 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚322 M@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚59049984 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 3216502174720 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚429 PIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚62 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚364060592295785 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚49257119 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚319821284372480 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 4654602240 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 3216502174720 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚49257117 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 4654602240 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚362998601089024 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚41945088 D;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚5264 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚319821215496192 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚61478912 _runtime‚3544.731929513 I5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5264 SFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚62581760 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 125829120 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 K>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59053056 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17104896 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 122905600 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚322 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 B6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚322 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚262 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654602240 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚44654906 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 5494538240 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚363024940531712 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚363024923426816 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 E8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚322 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚93911283 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17104896 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 3216733246464 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 G@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚67 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚30669804 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚319844511569920 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 845178880 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚61478912 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 D=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚60 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚4293 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 3216691298304 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 NGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚65 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 5494538240 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚223 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚363024923426816 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚262 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 3216733246464 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚49260672 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚316442959740928 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚364087067049361 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 125829120 _step‚1095 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚41948160 NADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚58387428 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚360877030440960 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚363024940531712 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚41948160 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 3209995260333 IBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚60 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 122905600 B9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚5264 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚44650613 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚429 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚322 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚319844574151680 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚4293 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚20643840 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 3401614410752 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 4654602240 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 4552982528 I@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚4876 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 5368709120 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚49260670 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚41832530 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚41937920 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚366241673778176 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚93911283 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚41832463 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚30669806 DDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚93915578 EDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521206 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 125829120 trainer/global_step‚13299 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚316442939097088 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 831258624 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚364087008661933 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 122306526 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚49260672 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚44650613 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 4552982528 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 3216691298304 H@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚665 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚41348068 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 5494538240µ_<Ð&ØÊM E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 _runtime‚3544.905913852 >5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5264 SFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚62581760 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 125829120 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 K>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59053056 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17104896 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 122905600 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚322 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 B6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚322 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚262 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654602240 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚44654906 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 5494538240 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚363024940531712 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚363024923426816 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 E8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚322 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚93911283 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17104896 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 3216733246464 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 G@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚67 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚30669804 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚319844511569920 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 845178880 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚61478912 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 D=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚60 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚4293 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 3216691298304 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 NGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚65 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 5494538240 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚223 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚363024923426816 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚262 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 3216733246464 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚49260672 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚316442959740928 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚364087067049361 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 125829120 _step‚1095 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚41948160 NADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚58387428 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚360877030440960 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚363024940531712 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚41948160 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 3209995260333 IBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚60 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 122905600 B9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚5264 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚44650613 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚429 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚322 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚319844574151680 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚4293 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚20643840 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 3401614410752 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 4654602240 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 4552982528 I@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚4876 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 5368709120 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚49260670 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚41832530 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚41937920 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚366241673778176 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚93911283 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚41832463 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚30669806 DDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚93915578 EDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521206 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 125829120 trainer/global_step‚13299 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚316442939097088 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 831258624 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚364087008661933 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 122306526 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚49260672 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚44650613 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 4552982528 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 3216691298304 H@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚665 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚41348068 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 5494538240 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 5368709120 NADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚59053056 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚93915578 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚44654906 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2Ç#ì2Ù- _wandbruntime‚3544  _runtime‚3544#»wÚñ &loss/total_loss‚2.0823734994794787 epoch‚33 trainer/global_step‚13299 % _timestamp‚1.7866621963765059e+09 _runtime‚3544.906337664 _step‚1096 0loss/FRt1/assignment_loss‚0.5926550271693563 0loss/FRt2/assignment_loss‚0.5640177420043181 0loss/FRt1/detection_loss‚0.45113874802050374 0loss/FRt2/detection_loss‚0.47456198228530067i§ª|Ûö 0loss/FRt1/detection_loss‚0.45113874802050374 0loss/FRt2/detection_loss‚0.47456198228530067 &loss/total_loss‚2.0823734994794787 epoch‚33 trainer/global_step‚13299 % _timestamp‚1.7866621963765059e+09 _runtime‚3544.906337664 _step‚1096 0loss/FRt1/assignment_loss‚0.5926550271693563 0loss/FRt2/assignment_loss‚0.5640177420043181ÈôbŠ–Ü‚(:‹ ¾šùÓ ˆˆá gpu.0.gpu‚64.0 gpu.0.memory‚12, gpu.0.memoryAllocated‚7.0313262939453125) gpu.0.memoryAllocatedBytes‚ 6039863296 gpu.0.temp‚36.0 gpu.0.powerWatts‚222.806' gpu.0.enforcedPowerLimitWatts‚500.0 gpu.0.powerPercent‚44.5612 gpu.0.smClock‚1410 gpu.0.memoryClock‚1593$ gpu.0.correctedMemoryErrors‚1955# gpu.0.uncorrectedMemoryErrors‚0»Ý‚(:’ ¾šùÓ¥ŒÕã$ memory_percent‚68.62068176269531% proc.memory.availableMB‚ 5141.1875) disk./.usagePercent‚32.71596979913936 disk.drbd2165.in‚0( disk.nvme0c0n1uñ“Éâ.out‚58955.31591796875 disk.sda1.in‚0" disk.nvme0n1.in‚43804.16015625# proc.memory.rssMB‚ 2362.08984375 cpu‚27.212039204745988 disk.dm-0.in‚43804.16015625 disk.sda3.in‚ 32534.8515625$ disk.nvme0c0n1.in‚43804.16015625 disk.sda.in‚ 32534.8515625 disk.sda2.in‚0 disk.sda2.out‚0& disk.nvme0n1.out‚58955.31591796875 disk.dm-1.out‚ 65.19140625 network.sent‚ 1.7276385e+07 network.recv‚ 6.583529e+06 disk.sda1.out‚0 disk.dm-1.in‚ 32534.8515625* proc.memory.percent‚14.417052268981934 proc.cpu.threads‚17% disk./.usageGB‚487.49159240722656 disk.drbd2165.out‚0# disk.dm-0.out‚58955.31591796875 disk.sda3.out‚ 65.19140625 disk.sda.out‚ 65.19140625wƒ–¬2Þ- _wandbruntime‚3557  _runtime‚3557¸&çËßÅ ' lr-AdamW/pg1‚0.00038731605344336764 ' lr-AdamW/pg2‚0.00038731605344336764 trainer/global_step‚13349 % _timestamp‚1.7866622088092766e+09 _runtime‚3557.339348204 _step‚10970;0XÐàÊ ' lr-AdamW/pg1‚0.00038731605344336764 ' lr-AdamW/pg2‚0.00038731605344336764 trainer/global_step‚13349 % _timestamp‚1.7866622088092766e+09 _runtime‚3557.339348204 _step‚1097ÉTMªe2á- _wandbruntime‚3557  _runtime‚3557fþ:áÁ'â»O @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 trainer/global_step‚13349 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚209 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 _step‚1098 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚367542824024064 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 3228046995456 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚44812428 IBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚79 F?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚60 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚364314735083520 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17104896 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 4552982528 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 122905600 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚44816715 BDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚367542764974080 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚41348020 @7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚5264 UHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚70973440 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚317572281008128 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 KDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚60 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚367542764974080 D;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚5264 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 3228088940544 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4289 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 3221326498329 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚41945088 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 122306526 IDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 5368709120 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚322 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚367542824024064 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚30777851 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 E8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚94247195 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 4552982528 IDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4876 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 4628521206 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚262 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚364314717978624 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚41945088 JBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚665 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚320985842653696 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 845178880 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚61478912 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚49434767 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚4287 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚317572260364288 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 5494538240 % _timestamp‚1.7866622088103094e+09 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚362160117284864 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚11202510 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 125829120 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚94251484 KBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚4876 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 4517085184 F>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚20643840 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 3413632619008 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚362160134324224 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 125829120 _runtime‚3557.343901615 PCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚59049984 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚320985913627136 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 5368709120 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚41980361 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 831258624 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚322 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚41980282•0ÞpÆ'ãÀO M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚41980282 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 trainer/global_step‚13349 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚209 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 _step‚1098 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚367542824024064 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 3228046995456 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚44812428 IBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚79 F?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚60 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚364314735083520 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17104896 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 4552982528 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 122905600 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚44816715 BDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚367542764974080 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚41348020 @7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚5264 UHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚70973440 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚317572281008128 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 KDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚60 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚367542764974080 D;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚5264 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 3228088940544 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4289 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 3221326498329 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚41945088 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 122306526 IDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 5368709120 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚322 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚367542824024064 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚30777851 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 E8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚94247195 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 4552982528 IDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4876 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 4628521206 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚262 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚364314717978624 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚41945088 JBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚665 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚320985842653696 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 845178880 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚61478912 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚49434767 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚4287 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚317572260364288 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 5494538240 % _timestamp‚1.7866622088103094e+09 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚362160117284864 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚11202510 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 125829120 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚94251484 KBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚4876 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 4517085184 F>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚20643840 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 3413632619008 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚362160134324224 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 125829120 _runtime‚3557.343901615 PCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚59049984 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚320985913627136 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 5368709120 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚41980361 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 831258624 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚322ÊÈÔº2ä- _wandbruntime‚3557  _runtime‚3557Vä4+Ë&åÅM ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 3413827484160 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 5368709120 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚367569377433600 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚364341057421312 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚44820008 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚52423680 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 _step‚1099 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚94258330 B9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚5264 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 3221598419009 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚44815715 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚94258330 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚49438320 NGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚72 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚20643840 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 NADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚58387428 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚41983366 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚4293 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚367569318380544 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 831258624 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 trainer/global_step‚13349 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚364341057421312 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 4552982528 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 3228278064128 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚429 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 3413879907840 >5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5264 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 K>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59053056 SFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚73067520 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 5494538240 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚364341040316416 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 5368709120 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚30780056 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 845178880 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17104896 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654602240 H@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚665 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚4293 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 125829120 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 3228278064128 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 4552982528 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚322 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚44820008 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚365407918536769 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚41348068 DDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚41983292 IBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚60 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚209 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚61478912 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521206 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚41948160 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚322 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚262 G@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚60 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚362186320117760 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚49438320 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚317595328380928 D=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚60 BDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚11203238 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚365407860149341 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 3228320012288 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚49438322 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 EDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 % _timestamp‚1.7866622089885864e+09 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 122905600 EDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 % _timestamp‚1.7866622089885864e+09 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 122905600 EDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚94258330 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚49438320 NGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚72 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚20643840 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 NADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚58387428 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚41983366 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚4293 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚367569318380544 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 831258624 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 trainer/global_step‚13349 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚364341057421312 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 4552982528 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 3228278064128 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚429 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 3413879907840 >5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5264 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 K>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59053056 SFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚73067520 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 5494538240 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚364341040316416 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 5368709120 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚30780056 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 845178880 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17104896 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654602240 H@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚665 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚4293 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 125829120 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 3228278064128 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 4552982528 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚322 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚44820008 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚365407918536769 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚41348068 DDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚41983292 IBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚60 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚209 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚61478912 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521206 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚41948160 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚322 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚262 G@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚60 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚362186320117760 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚49438320 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚317595328380928 D=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚60 BDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚11203238 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚365407860149341 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0˼£˜ð2ç- _wandbruntime‚3557  _runtime‚3557¼e uèï _runtime‚3557.519573951 0loss/FRt1/assignment_loss‚0.6801023218114173 0loss/FRt2/assignment_loss‚0.5834622556978963 % _timestamp‚1.7866622089897122e+09 _step‚1100 /loss/FRt1/detection_loss‚0.4959992199996583 0loss/FRt2/detection_loss‚0.48875009842118694 %loss/total_loss‚2.248313895930159 epoch‚33 trainer/global_step‚13349Å Æ‘zéô _step‚1100 /loss/FRt1/detection_loss‚0.4959992199996583 0loss/FRt2/detection_loss‚0.48875009842118694 %loss/total_loss‚2.248313895930159 epoch‚33 trainer/global_step‚13349 _runtime‚3557.519573951 0loss/FRt1/assignment_loss‚0.6801023218114173 0loss/FRt2/assignment_loss‚0.5834622556978963 % _timestamp‚1.7866622089897122e+09Ìýt –ê‚(:‹ ÍšùÓ‘èìà gpu.0.gpu‚62.0 gpu.0.memory‚12, gpu.0.memoryAllocated‚7.0313262939453125) gpu.0.memoryAllocatedBytes‚ 6039863296 gpu.0.temp‚36.0 gpu.0.powerWatts‚222.806' gpu.0.enforcedPowerLimitWatts‚500.0 gpu.0.powerPercent‚44.5612 gpu.0.smClock‚1410 gpu.0.memoryClock‚1593$ gpu.0.correctedMemoryErrors‚1955# gpu.0.uncorrectedMemoryErrors‚0ºjˆ ë‚(:• ÍšùÓ£›£ã* disk./.usagePercent‚32.716032008970174# disk.dm-0.out‚59001.47216796875 disk.sda3.out‚ 65.37109375 disk.sda.in‚ 32534.8515625 disk.sda1.in‚0* proc.memory.percent‚14.417052268981934 proc.cpu.threads‚17( disk.nvme0c0n1.out‚59001.47216796875 disk.dm-1.out‚ 65.37109375( proc.memory.availableMB‚ 5141.1328125# proc.memory.rssMB‚ 2362.08984375 cpu‚27.211718116985605 network.recv‚ 6.616298e+06 disk.drbd2165.in‚0 disk.drbd2165.out‚0 disk.sda1.out‚0 disk.sda2.out‚0" disk.nvme0n1.in‚43804.16015625 disk.dm-1.in‚ 32534.8515625$ memory_percent‚68.62101554870605 network.sent‚ 1.7351498e+07$ disk./.usageGB‚487.4925193786621 disk.dm-0.in‚43804.16015625 disk.sda3.in‚ 32534.8515625$ disk.nvme0c0n1.in‚43804.16015625 disk.sda.out‚ 65.37109375 disk.sda2.in‚0& disk.nvme0n1.out‚59001.47216796875é4²2ì- _wandbruntime‚3570  _runtime‚3570Ã'&GÉíà & lr-AdamW/pg1‚0.0003820665854350111 & lr-AdamW/pg2‚0.0003820665854350111 trainer/global_step‚13399 % _timestamp‚1.7866622219039307e+09 _runtime‚3570.433946671 _step‚1101àuzÎîÈ % _timestamp‚1.7866622219039307e+09 _runtime‚3570.433946671 _step‚1101 & lr-AdamW/pg1‚0.0003820665854350111 & lr-AdamW/pg2‚0.0003820665854350111 trainer/global_step‚13399Ímž­ø2ï- _wandbruntime‚3570  _runtime‚3570²ó6Á'ð»O cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 3425916859392 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚322 F?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚60 UHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚66779136 _runtime‚3570.438901159 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚318724649648128 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 BDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚368870468629504 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17104896 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚368870468629504 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚363469424001024 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 4517085184 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 IDeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4289 IBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚74 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚41945088 JBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚665 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 845178880 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚49612417 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚41945088 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 125829120 % _timestamp‚1.7866622219048705e+09 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 3239633761280 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚368870527679488 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚44981817 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚11243434 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚322150499728384 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 3232888308937 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚322 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 4654602240 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚365630834868224 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚42131535 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚4289 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 4552982528 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 5368709120 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 4654602240 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 122306526 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚11243362 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 5368709120 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 3232929656957 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 5494538240 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 125829120 F>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚4287 G>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4876 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚366702353657981 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 122905600 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚49612417 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚322150566507520 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 5494538240 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚365630851973120 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚363469406961664 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 5368709120 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚94594236 PIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚72 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚262 KDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚60 E8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚94589947 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚365630834868224 KBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚4876 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚30888101 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚322 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚49612419 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 3425870724096 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚318724629004288#7ÐÆ'ñÀO GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚44981817 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 4552982528 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 3239675706368 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚42131461 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 M@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚59049984 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 3239675706368 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 3239633761280 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚223 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 125829120 IBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚60 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4289 IBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚74 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚41945088 JBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚665 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 845178880 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚49612417 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚41945088 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 125829120 % _timestamp‚1.7866622219048705e+09 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 3239633761280 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚368870527679488 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚44981817 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚11243434 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚322150499728384 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 3232888308937 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚322 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 4654602240 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚365630834868224 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚42131535 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚4289 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 4552982528 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 5368709120 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 4654602240 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 122306526 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚11243362 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 5368709120 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 3232929656957 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 5494538240 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 125829120 F>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚4287 G>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4876 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚366702353657981 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 122905600 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚49612417 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚322150566507520 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 5494538240 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚365630851973120 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚363469406961664 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 5368709120 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚94594236 PIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚72 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚262 KDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚60 E8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚94589947 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚365630834868224 KBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚4876 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚30888101 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚322 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚49612419 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 3425870724096 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚318724629004288 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 3425916859392 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚322 F?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚60 UHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚66779136 _runtime‚3570.438901159 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚318724649648128 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 BDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚368870468629504 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17104896 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚368870468629504 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚363469424001024 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 4517085184 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 I6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚322 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚262 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚209 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚262 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚49615970 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17104896 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚322173859670016 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚50326528 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 122306526 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚44980817 EDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59053056 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚94601082 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚44980817 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚41348068 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚49615972 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 4552982528 H@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚665 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 BDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚42134436 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚365657174310912 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚365657157206016 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚4293 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 831258624 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚41948160 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚49615970 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 NADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚58387428 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 125829120 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 D=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚60 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚94596787 P8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚322 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 SFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚70970368 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 5494538240 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 125829120 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚4295 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚368897022035968 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 4552982528 NGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚70 >5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5264 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚44985110 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654602240 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚11244202 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521206 % _timestamp‚1.7866622220675776e+09 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚42134508 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 125829120 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚429 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚365657174310912 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚322 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚30890304 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 122905600 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚61478912 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 5368709120 IBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚60 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 3239906778112 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚49615972 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 5368709120 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚363495609794560 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 I@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚4876 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚363495592755200 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚41948160 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 3239864829952 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 4517085184 trainer/global_step‚13399 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 4654602240 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚365657157206016 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 3426162649088 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 5494538240 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 5368709120 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 _step‚1103 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚30890306 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 NADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚59053056 B9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚5264 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 3233160229617 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚44985110 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚20643840å;ÜÞÐ&ôÊM QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 5494538240 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 125829120 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚4295 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚368897022035968 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 4552982528 NGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚70 >5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5264 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚44985110 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654602240 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚11244202 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521206 % _timestamp‚1.7866622220675776e+09 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚42134508 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 125829120 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚429 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚365657174310912 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚322 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚30890304 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 122905600 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚61478912 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 5368709120 IBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚60 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 3239906778112 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚49615972 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 5368709120 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚363495609794560 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 I@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚4876 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚363495592755200 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚41948160 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 3239864829952 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 4517085184 trainer/global_step‚13399 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 4654602240 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚365657157206016 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 3426162649088 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 5494538240 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 5368709120 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 _step‚1103 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚30890306 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 NADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚59053056 B9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚5264 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 3233160229617 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚44985110 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚20643840 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17104896 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 845178880 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚366728711636749 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 5494538240 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚262 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚4293 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚368897081089024 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚322173788699648 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚318747676377088 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 3426112322560 G@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚60 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚322 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚262 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚209 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚262 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚49615970 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17104896 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚322173859670016 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚50326528 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 122306526 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚44980817 EDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59053056 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚94601082 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚44980817 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚41348068 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚49615972 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 4552982528 H@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚665 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 BDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚42134436 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚365657174310912 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚365657157206016 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚4293 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 831258624 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚41948160 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚49615970 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 NADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚58387428 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 125829120 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 D=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚60 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚94596787 P8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚322 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 SFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚70970368Ïiw42õ- _wandbruntime‚3570  _runtime‚3570 a?töî /loss/FRt1/detection_loss‚0.4791728542533416 /loss/FRt2/detection_loss‚0.4687098810206766 trainer/global_step‚13399 % _timestamp‚1.7866622220685973e+09 0loss/FRt1/assignment_loss‚0.5631519670019308 0loss/FRt2/assignment_loss‚0.4796351675265568 %loss/total_loss‚1.990669869802506 epoch‚33 _runtime‚3570.598422558 _step‚1104bÚK)y÷ó 0loss/FRt1/assignment_loss‚0.5631519670019308 0loss/FRt2/assignment_loss‚0.4796351675265568 %loss/total_loss‚1.990669869802506 epoch‚33 _runtime‚3570.598422558 _step‚1104 /loss/FRt1/detection_loss‚0.4791728542533416 /loss/FRt2/detection_loss‚0.4687098810206766 trainer/global_step‚13399 % _timestamp‚1.7866622220685973e+09Ъ5•2ø- _wandbruntime‚3582  _runtime‚3582qZËùÅ % _timestamp‚1.7866622343281622e+09 _runtime‚3582.858169959 _step‚1105 ' lr-AdamW/pg1‚0.00037684074910233255 ' lr-AdamW/pg2‚0.00037684074910233255 trainer/global_step‚13449@òl*ÐúÊ ' lr-AdamW/pg2‚0.00037684074910233255 trainer/global_step‚13449 % _timestamp‚1.7866622343281622e+09 _runtime‚3582.858169959 _step‚1105 ' lr-AdamW/pg1‚0.00037684074910233255ÑÞp¬ß2û- _wandbruntime‚3582  _runtime‚3582Š@Ü‚Á'ü»O G>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4876 IDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚370198172284928 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 5368709120 _step‚1106 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚370198231334912 D;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚5264 PIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚78 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚323315226555904 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 4517085184 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚41945088 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 3251220527104 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 845178880 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚61478912 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 5368709120 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 3251262472192 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 125829120 _runtime‚3582.862875071 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 PCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚59049984 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17104896 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚20643840 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 831258624 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 3438162132480 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚366946951757824 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4289 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 122905600 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 4552982528 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚364778713677824 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚366946951757824 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚30998351 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 3244450119545 E8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚94932699 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚49790067 F>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚319877018288128 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚41348020 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 125829120 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 4552982528 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚370198231334912 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚429 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 5494538240 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚322 IBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚60 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚45146919 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 122306526 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚49790067 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 3438208267776 BDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4876 IDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚370198172284928 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 5368709120 _step‚1106 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚370198231334912 D;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚5264 PIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚78 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚323315226555904 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 4517085184 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚41945088 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 3251220527104 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 845178880 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚61478912 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 5368709120 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 3251262472192 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 125829120 _runtime‚3582.862875071 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 PCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚59049984 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17104896 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚20643840 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 831258624 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 3438162132480 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚366946951757824 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4289 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 122905600 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 4552982528 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚364778713677824 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚366946951757824 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚30998351 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 3244450119545 E8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚94932699 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚49790067 F>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚319877018288128 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚41348020 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 125829120 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 4552982528 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚370198231334912 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚429 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 5494538240 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚322 IBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚60 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚45146919 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 122306526 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚49790067 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 3438208267776 BDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚49793622 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 4517085184 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚94939539 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚319900045017088 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17104896 I@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚4876 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚11285070 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 3244722040225 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚49793620 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 122905600 BDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚94943834 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 5368709120 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 125829120 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚42285626 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚61478912 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521206 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚364804899471360 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚322 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚45150212 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚45145919 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 831258624 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚322 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚209 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 845178880 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚20643840 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 122905600 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 D=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚60 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 125829120 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚366973274095616 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17104896 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚322 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 122306526 _runtime‚ 3583.02557251 NGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚66 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚370224784744448 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 3251451595776 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 G@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚60 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 5368709120 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 D5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5264 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 3251493543936 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚42285558 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚319900065660928 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 EDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59053056 P8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 5368709120 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚364804882432000 % _timestamp‚1.7866622344951286e+09 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚370224725691392 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚11285004 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚322 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚262 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654602240 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 4552982528 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚4293 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 B9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚5264 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚323338517621248 SFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚66776064 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 3438405828096 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚49793620 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚05ÃÏ&€ ÉM VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 5368709120 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 D5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5264 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 3251493543936 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚42285558 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚319900065660928 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 EDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59053056 P8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 5368709120 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚364804882432000 % _timestamp‚1.7866622344951286e+09 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚370224725691392 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚11285004 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚322 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚262 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654602240 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 4552982528 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚4293 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 B9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚5264 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚323338517621248 SFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚66776064 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 3438405828096 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚49793620 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚41948160 EDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚49793622 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 4517085184 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚94939539 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚319900045017088 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17104896 I@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚4876 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚11285070 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 3244722040225 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚49793620 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 122905600 BDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚94943834 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 5368709120 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 125829120 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚42285626 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚61478912 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521206 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚364804899471360 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚322 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚45150212 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚45145919 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 831258624 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚322 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚209 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 845178880 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚20643840 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 122905600 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 D=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚60 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 125829120 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚366973274095616 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17104896 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚322 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 122306526 _runtime‚ 3583.02557251 NGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚66 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚370224784744448 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 3251451595776 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 G@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚60ÓõòÙä2 - _wandbruntime‚3583  _runtime‚3583´æ3Us‚ í 0loss/FRt1/assignment_loss‚0.5584970057192592 1loss/FRt2/assignment_loss‚0.44332550366670603 /loss/FRt1/detection_loss‚0.5034890257698154 &loss/total_loss‚1.9560760212198807 trainer/global_step‚13449 _step‚1108 ,loss/FRt2/detection_loss‚0.4507644860641 epoch‚33 % _timestamp‚1.7866622344961689e+09 _runtime‚3583.026118703ˆL„¢xƒ ò ,loss/FRt2/detection_loss‚0.4507644860641 epoch‚33 % _timestamp‚1.7866622344961689e+09 _runtime‚3583.026118703 0loss/FRt1/assignment_loss‚0.5584970057192592 1loss/FRt2/assignment_loss‚0.44332550366670603 /loss/FRt1/detection_loss‚0.5034890257698154 &loss/total_loss‚1.9560760212198807 trainer/global_step‚13449 _step‚1108ÔÅ*¦–„ ‚(:‹ ÜšùÓ±Šá gpu.0.gpu‚64.0 gpu.0.memory‚12, gpu.0.memoryAllocated‚7.0313262939453125) gpu.0.memoryAllocatedBytes‚ 6039863296 gpu.0.temp‚37.0 gpu.0.powerWatts‚216.883' gpu.0.enforcedPowerLimitWatts‚500.0 gpu.0.powerPercent‚43.3766 gpu.0.smClock‚1410 gpu.0.memoryClock‚1593$ gpu.0.correctedMemoryErrors‚1955# gpu.0.uncorrectedMemoryErrors‚0þY( ‰… ‚(:þ ÜšùÓןÓã disk.sda.in‚ 32557.71875" disk.nvme0n1.in‚43805.04296875 disk.drbd2165.in‚0$ memory_percent‚68.62554550170898# proc.memory.rssMB‚ 2362.08984375' proc.memory.availableMB‚ 5140.390625* proc.memory.percent‚14.417052268981934 cpu‚27.211318628209575 proc.cpu.threads‚17 disk.sda1.out‚0 disk.sda2.out‚0 network.sent‚ 1.7426572e+07) disk./.usagePercent‚32.71603226497771$ disk./.usageGB‚487.4925231933594 disk.sda3.in‚ 32557.71875 disk.sda3.out‚ 65.640625$ disk.nvme0c0n1.in‚43805.04296875% disk.nvme0c0n1.out‚59008.88671875 disk.sda1.in‚0# disk.nvme0n1.out‚59008.88671875 disk.dm-1.in‚ 32557.71875 disk.dm-1.out‚ 65.640625 disk.drbd2165.out‚0 disk.dm-0.in‚43805.04296875 disk.dm-0.out‚59008.88671875 disk.sda.out‚ 65.640625 disk.sda2.in‚0 network.recv‚ 6.648838e+061¹22† - _wandbruntime‚3595  _runtime‚3595=,¥É‡ à _runtime‚3595.777918524 _step‚1109 & lr-AdamW/pg1‚0.0003716388800910345 & lr-AdamW/pg2‚0.0003716388800910345 trainer/global_step‚13499 % _timestamp‚1.7866622472458174e+09VŸí-Έ È _step‚1109 & lr-AdamW/pg1‚0.0003716388800910345 & lr-AdamW/pg2‚0.0003716388800910345 trainer/global_step‚13499 % _timestamp‚1.7866622472458174e+09 _runtime‚3595.777918524ÕÔ¿®2‰ - _wandbruntime‚3595  _runtime‚3595øò6'Š ºO UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 5368709120 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚324479799877632 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚49967717 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚429 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚41348020 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚45312021 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 122905600 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 5368709120 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 3450475534336 B™StatsMonitor.on_train_batch_start/active.all.freed‚95275451 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚42433602 PIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚68 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚322 UHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚62584832 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 831258624 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚61478912 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚95279740 IDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚371525875940352 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 3262849238016 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚369344056632797 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 F?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚60 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚371525934990336 D;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚5264 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 5494538240 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 125829120 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚322 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17104896 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 122905600 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚262 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17104896 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚321029386928128 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚262 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚4287 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚20643840 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 3450433593344 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚4287 KBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚4876 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 4654602240 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 3262849238016 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 M@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚59049984 PCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚59049984 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚324479862462464 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 PCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚58387380 trainer/global_step‚13499 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚45307734 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚31108601 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 4628521206 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚4289 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚368263068647424 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚31108599 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 @7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚5264 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚371525875940352 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 4552982528 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚41945088 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚49967717 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚11325001 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚223 G>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4876 IBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚70 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚322 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚262 $ _timestamp‚1.786662247246526e+09 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚368263085752320 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚209 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 3262807292928 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4289 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚49967719 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚11324933 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚369343998245417 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚366087986315264 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 3262807292928 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚41940992 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 _runtime‚3595.778061434 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚322 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 _step‚1110 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚371525934990336 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 4654602240 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚45312021 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚42433532 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 845178880 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 5494538240 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚45307734 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 125829120 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 4517085184.˜Ä!Å'‹ ¿O ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 _runtime‚3595.778061434 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚322 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 _step‚1110 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚371525934990336 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 4654602240 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚45312021 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚42433532 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 845178880 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 5494538240 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚45307734 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 125829120 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 4517085184 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 5368709120 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚324479799877632 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚49967717 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚429 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚41348020 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚45312021 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 122905600 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 5368709120 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 3450475534336 BDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚371525875940352 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 3262849238016 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚369344056632797 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 F?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚60 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚371525934990336 D;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚5264 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 5494538240 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 125829120 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚322 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17104896 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 122905600 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚262 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17104896 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚321029386928128 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚262 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚4287 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚20643840 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 3450433593344 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚4287 KBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚4876 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 4654602240 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 3262849238016 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 M@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚59049984 PCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚59049984 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚324479862462464 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 PCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚58387380 trainer/global_step‚13499 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚45307734 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚31108601 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 4628521206 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚4289 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚368263068647424 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚31108599 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 @7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚5264 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚371525875940352 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 4552982528 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚41945088 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚49967717 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚11325001 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚223 G>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4876 IBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚70 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚322 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚262 $ _timestamp‚1.786662247246526e+09 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚368263085752320 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚209 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 3262807292928 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4289 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚49967719 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚11324933 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚369343998245417 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚366087986315264 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 3262807292928 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚41940992ÖHŒq2Œ - _wandbruntime‚3595  _runtime‚3595ÇHÚË& ÅM D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚368289408090112 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚368289390985216 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚11325733 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 3256283850833 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 3263038361600 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 4552982528 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚31110806 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚20643840 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521206 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚368289390985216 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 3263038361600 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 845178880 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚44035072 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚369370414611565 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 125829120 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 125829120 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 3263080309760 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚322 E6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚322 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚371552488399872 I@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚4876 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚61478912 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚369370472998993 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚45311021 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17104896 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚95286586 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚322 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚262 _step‚1111 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚45315314 IBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚60 _runtime‚3595.951263294 >5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5264 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚368289408090112 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚45315314 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚223 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 5494538240 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚322 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚95286586 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚371552488399872 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654602240 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚49971270 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚429 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 122905600 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚11325789 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 3263080309760 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 BDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 K>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59053056 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚4293 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚366114172108800 EDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚42436537 PDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 D=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚60 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 4552982528 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 122905600 G@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚60 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 125829120 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚4293¦ø&‚Ž ÊM C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚322 E6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚322 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚371552488399872 I@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚4876 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚61478912 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚369370472998993 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚45311021 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17104896 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚95286586 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚322 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚262 _step‚1111 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚45315314 IBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚60 _runtime‚3595.951263294 >5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5264 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚368289408090112 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚45315314 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚223 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 5494538240 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚322 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚95286586 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚371552488399872 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654602240 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚49971270 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚429 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 122905600 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚11325789 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 3263080309760 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 BDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 K>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59053056 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚4293 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚366114172108800 EDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚42436537 PDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 D=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚60 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 4552982528 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 122905600 G@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚60 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 125829120 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚4293 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚368289408090112 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚368289390985216 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚11325733 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 3256283850833 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 3263038361600 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 4552982528 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚31110806 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚20643840 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521206 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚368289390985216 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 3263038361600 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 845178880 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚44035072 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚369370414611565 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 125829120 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 125829120 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 3263080309760×<û;2 - _wandbruntime‚3595  _runtime‚3595Çí¦{t î /loss/FRt1/assignment_loss‚0.607235891000958 0loss/FRt2/assignment_loss‚0.4512467935768769 /loss/FRt1/detection_loss‚0.5050143749342357 &loss/total_loss‚2.0269427642056757 trainer/global_step‚13499 _runtime‚3595.951986692 /loss/FRt2/detection_loss‚0.4634457046936052 epoch‚33 % _timestamp‚1.7866622474220588e+09 _step‚11121qØ!y‘ ó /loss/FRt1/assignment_loss‚0.607235891000958 0loss/FRt2/assignment_loss‚0.4512467935768769 /loss/FRt1/detection_loss‚0.5050143749342357 &loss/total_loss‚2.0269427642056757 trainer/global_step‚13499 _runtime‚3595.951986692 /loss/FRt2/detection_loss‚0.4634457046936052 epoch‚33 % _timestamp‚1.7866622474220588e+09 _step‚1112ØWt '–’ ‚(:‹ ëšùÓÄáá gpu.0.gpu‚63.0 gpu.0.memory‚12, gpu.0.memoryAllocated‚7.0313262939453125) gpu.0.memoryAllocatedBytes‚ 6039863296 gpu.0.temp‚36.0 gpu.0.powerWatts‚204.557' gpu.0.enforcedPowerLimitWatts‚500.0 gpu.0.powerPercent‚40.9114 gpu.0.smClock‚1410 gpu.0.memoryClock‚1593$ gpu.0.correctedMemoryErrors‚1955# gpu.0.uncorrectedMemoryErrors‚0jÃ’‘“ ‚(:† ëšùÓ‹ï‚ã) proc.memory.availableMB‚ 5140.30078125 proc.cpu.threads‚17) disk./.usagePercent‚32.73294642675743 disk.sda2.out‚0 disk.nvme0n1.in‚ 43805.4375 disk.dm-1.in‚ 33588.8046875 disk.drbd2165.in‚0 disk.nvme0c0n1.in‚ 43805.4375# proc.memory.rssMB‚ 2362.08984375 cpu‚27.211075337402143 disk.dm-1.out‚ 65.8515625# disk.dm-0.out‚59053.47802734375( disk.nvme0c0n1.out‚59053.47802734375 disk.sda.out‚ 65.8515625$ memory_percent‚68.62609386444092* proc.memory.percent‚14.417052268981934 disk.drbd2165.out‚0 network.sent‚ 1.751775e+07 disk.sda1.out‚0& disk.nvme0n1.out‚59053.47802734375 disk.dm-0.in‚ 43805.4375 disk.sda3.in‚ 33592.5546875 disk.sda3.out‚ 65.8515625 disk.sda.in‚ 33592.5546875 network.recv‚ 6.676795e+06% disk./.usageGB‚487.74455642700195 disk.sda1.in‚0 disk.sda2.in‚0ƒexÐ2” - _wandbruntime‚3608  _runtime‚3608[…ǘÉ• à _step‚1113 & lr-AdamW/pg1‚0.0003664613125074424 & lr-AdamW/pg2‚0.0003664613125074424 trainer/global_step‚13549 % _timestamp‚1.7866622598174393e+09 _runtime‚3608.349903419ƒÌΖ È _step‚1113 & lr-AdamW/pg1‚0.0003664613125074424 & lr-AdamW/pg2‚0.0003664613125074424 trainer/global_step‚13549 % _timestamp‚1.7866622598174393e+09 _runtime‚3608.349903419Ù÷áš2— - _wandbruntime‚3608  _runtime‚3608t£qsÀ'˜ ºO RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 122905600 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚369579185537024 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 M@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚59049984 IDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4876 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚262 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 3274394058752 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚50145367 D;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚5264 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 5368709120 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚4287 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17104896 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4289 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚50145369 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚322 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 4628521206 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 5494538240 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚429 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 5368709120 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚95622492 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚372853638645760 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚325644470583808 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 3267615088781 IBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚60 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 3274394058752 BDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚372853579595776 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 3274436003840 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚31218851 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚20643840 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 4517085184 F>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚372853579595776 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 122306526øB0&Å'™ ¿O % _timestamp‚1.7866622598181117e+09 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 122905600 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚322181755568128 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 KDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚60 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 4552982528 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚31218849 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 _runtime‚ 3608.34998828 IDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4876 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚262 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 3274394058752 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚50145367 D;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚5264 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 5368709120 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚4287 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17104896 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4289 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚50145369 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚322 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 4628521206 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 5494538240 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚429 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 5368709120 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚95622492 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚372853638645760 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚325644470583808 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 3267615088781 IBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚60 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 3274394058752 BDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚372853579595776 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 3274436003840 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚31218851 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚20643840 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 4517085184 F>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚372853579595776 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 122306526 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 122905600 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚369579185537024 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 M@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚59049984 IX› ÃM D=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚60 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 NGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚61 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 5368709120 PDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚95629338 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 _runtime‚3608.510996029 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚372880192055296 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚369605507874816 G@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚63 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚42587442 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚322 >5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5264 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17104896 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 5494538240 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 5368709120 trainer/global_step‚13549 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654602240 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚209 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚325667767017984 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚322204782297088 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 831258624 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚61478912 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚262 K>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59053056 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚50148920 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 3267804313373 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚262 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚95625043 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 4654602240 NADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚59053056 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 3463022464512 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚45476123 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 122905600 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 3274625127424 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 845178880 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚370691266098973 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚4295 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 3274667075584 O@DeviceStatsMonitor.on_train_batch‘/”sq_end/reserved_bytes.all.current‚ 5494538240 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚45480416 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 4552982528 I@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚4876 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚322204802940928 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521206 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 3267845661441 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚322 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚369605507874816 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚325667825405440 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚41948160 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 125829120 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚369605524979712 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚11366449 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚367423461785600 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚367423478824960 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚50148922 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚369605524979712 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚50148922 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 EDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚372880133002240 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 125829120 H@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚665 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 5494538240 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚262 D8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 # _timestamp‚1.78666225998072e+09 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚429 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚4293 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 4552982528 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚4293 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚370691324486401 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 4517085184 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚41948160 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚20643840 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚11366388 G@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚60 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚50148920 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚95625043 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚223 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚37743616 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚322 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 EDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59053056 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚50148920 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 3267804313373 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚262 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚95625043 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 4654602240 NADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚59053056 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 3463022464512 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚45476123 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 122905600 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 3274625127424 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 845178880 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚370691266098973 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚4295 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 3274667075584 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 5494538240 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚45480416 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 4552982528 I@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚4876 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚322204802940928 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521206 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 3267845661441 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚322 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚369605507874816 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚325667825405440 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚41948160 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 125829120 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚369605524979712 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚11366449 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚367423461785600 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚367423478824960 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚50148922 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚369605524979712 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚50148922 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 EDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚372880133002240 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 125829120 H@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚665 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 5494538240 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚262 D8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 # _timestamp‚1.78666225998072e+09 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚429 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚4293 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 4552982528 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚4293 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚370691324486401 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 4517085184 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚41948160 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚20643840 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚11366388 G@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚60 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚50148920 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚95625043 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚223 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚37743616 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚322 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 EDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚95629338 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 _runtime‚3608.510996029 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚372880192055296 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚369605507874816 G@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚63 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚42587442 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚322 >5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5264 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17104896 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 5494538240 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 5368709120 trainer/global_step‚13549 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654602240 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚209 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚325667767017984 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚322204782297088 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 831258624 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚61478912 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚262ÛŽ'Çþ2 - _wandbruntime‚3608  _runtime‚3608ý,ï?už ï epoch‚33 trainer/global_step‚13549 % _timestamp‚1.7866622599817035e+09 _runtime‚3608.511488283 0loss/FRt1/assignment_loss‚0.5847167543912629 1loss/FRt2/assignment_loss‚0.47231230681662545 .loss/FRt1/detection_loss‚0.502599057117099 /loss/FRt2/detection_loss‚0.4843885313988832 &loss/total_loss‚2.0440166497238703 _step‚1116O0O±zŸ ô 0loss/FRt1/assignment_loss‚0.5847167543912629 1loss/FRt2/assignment_loss‚0.47231230681662545 .loss/FRt1/detection_loss‚0.502599057117099 /loss/FRt2/detection_loss‚0.4843885313988832 &loss/total_loss‚2.0440166497238703 _step‚1116 epoch‚33 trainer/global_step‚13549 % _timestamp‚1.7866622599817035e+09 _runtime‚3608.511488283Ü   ‚(:• úšùÓŠÜ€á gpu.0.gpu‚37.0 gpu.0.memory‚8, gpu.0.memoryAllocated‚7.0313262939453125) gpu.0.memoryAllocatedBytes‚ 6039863296 gpu.0.temp‚34.0 gpu.0.powerWatts‚119.843' gpu.0.enforcedPowerLimitWatts‚500.0) gpu.0.powerPercent‚23.968600000000002 gpu.0.smClock‚1410 gpu.0.memoryClock‚1593$ gpu.0.correctedMemoryErrors‚1955# gpu.0.uncorrectedMemoryErrors‚0êdz¡ ‚(:„ úšùÓ¼¸µã disk.drbd2165.in‚0 disk.dm-0.in‚ 43832.59375 disk.sda1.out‚0( proc.memory.availableMB‚ 5140.0234375 cpu‚27.21054470835515 proc.cpu.threads‚17 network.recv‚ 6.728881e+06 disk.sda2.out‚0& disk.nvme0n1.out‚59060.29931640625 disk.dm-1.in‚33679.45703125 disk.sda3.in‚33679.45703125! disk.nvme0c0n1.in‚ 43832.59375( disk.nvme0c0n1.out‚59060.29931640625$ memory_percent‚68.62778663635254) disk./.usagePercent‚32.73294156261428$ disk./.usageGB‚487.7444839477539 disk.dm-1.out‚66.40625# disk.dm-0.out‚59060.29931640625 disk.sda3.out‚66.40625 disk.sda.out‚66.40625# proc.memory.rssMB‚ 2362.08984375* proc.memory.percent‚14.417052268981934 network.sent‚ 1.7602635e+07 disk.sda2.in‚0 disk.nvme0n1.in‚ 43832.59375 disk.drbd2165.out‚0 disk.sda.in‚33679.45703125 disk.sda1.in‚0ßi`2¢ - _wandbruntime‚3621  _runtime‚3621f[#Ë£ Å _runtime‚3621.199656763 _step‚1117 ' lr-AdamW/pg1‚0.00036130837889704754 ' lr-AdamW/pg2‚0.00036130837889704754 trainer/global_step‚13599 % _timestamp‚1.7866622726696458e+09ÖTFòФ Ê ' lr-AdamW/pg1‚0.00036130837889704754 ' lr-AdamW/pg2‚0.00036130837889704754 trainer/global_step‚13599 % _timestamp‚1.7866622726696458e+09 _runtime‚3621.199656763 _step‚1117Ýùz2¥ - _wandbruntime‚3621  _runtime‚3621Ëû€Á'¦ »O `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚370895319531520 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚41945088 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚368706582708224 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 125829120 IDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 122905600 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 4552982528 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 5368709120 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚374181283251200 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚50323017 UHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚70973440 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 845178880 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚323334103564288 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 4628521206 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚50323017 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚4287 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚370895302426624 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚262 E8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚95960955 D;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚5264 trainer/global_step‚13599 _step‚1118 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚50323019 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚61478912 BDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4876 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚374181283251200 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚50329600 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚262 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4289 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚31329099 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚322 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 4552982528 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚370895302426624 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚323334124208128 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚368706565668864 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚11406120 PCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚58387380 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 3279176899389 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 5494538240 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 3475055057920 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 4517085184 KBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚4876 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚95965244 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 125829120 KDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚60 C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚4289 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚45642225 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚42735293 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 IDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4876 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚374181283251200 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚50329600 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚262 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4289 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚31329099 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚322 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 4552982528 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚370895302426624 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚323334124208128 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚368706565668864 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚11406120 PCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚58387380 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 3279176899389 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 5494538240 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 3475055057920 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 4517085184 KBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚4876 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚95965244 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 125829120 KDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚60 C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚4289 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚45642225 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚42735293 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 IDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 122905600 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 4552982528 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 5368709120 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚374181283251200 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚50323017 UHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚70973440 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 845178880 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚323334103564288 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 4628521206 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚50323017 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚4287 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚370895302426624 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚262 E8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚95960955 D;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚5264 trainer/global_step‚13599 _step‚1118 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚50323019 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚61478912 BDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521206 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 5494538240 _step‚1119 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚374207836657664 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚45641225 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚223 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 3475301658624 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚372012175973809 D=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚60 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚368732768501760 K>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59053056 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚95967795 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚368732751462400 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 845178880 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚370921641869312 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚370921624764416 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 BDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚95972090 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚50326572 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚31331304 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 125829120 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚429 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚31331306 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚11406939 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚322 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚372012117586381 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚95972090 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚50326572 D5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5264 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 122905600 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚45645518 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚4293 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 G@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚60 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚4293 B9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚5264 NGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚75 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚326832402269184 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 831258624 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 4552982528 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚11407014 % _timestamp‚1.7866622728409574e+09 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚4295 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚374207895710720 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 3286211893248 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚50326570 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚323357150937088 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 H@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚665 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 5494538240 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 3286253841408 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 3286211893248 G@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚77 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚42738243 NADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚58387428 EDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚262 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚50326570 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚41948160 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 5368709120 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚262 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 4654602240 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654602240 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 125829120 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 5368709120 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚45645518 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚370921641869312 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚370921624764416 I@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚4876 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚323357171580928 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 125829120 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚322ýæÜàÐ&ª ÊM [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚370921641869312 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚370921624764416 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 BDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚95972090 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚50326572 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚31331304 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 125829120 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚429 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚31331306 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚11406939 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚322 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚372012117586381 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚95972090 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚50326572 D5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5264 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 122905600 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚45645518 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚4293 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 G@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚60 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚4293 B9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚5264 NGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚75 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚326832402269184 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 831258624 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 4552982528 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚11407014 % _timestamp‚1.7866622728409574e+09 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚4295 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚374207895710720 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 3286211893248 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚50326570 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚323357150937088 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 H@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚665 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 5494538240 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 3286253841408 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 3286211893248 G@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚77 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚42738243 NADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚58387428 EDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚262 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚50326570 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚41948160 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 5368709120 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚262 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 4654602240 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654602240 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 125829120 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 5368709120 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚45645518 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚370921641869312 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚370921624764416 I@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚4876 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚323357171580928 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 125829120 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚322 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17104896 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 3286253841408 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 trainer/global_step‚13599 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚41948160 EDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521206 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 5494538240 _step‚1119 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚374207836657664 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚45641225 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚223 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 3475301658624 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚372012175973809 D=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚60 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚368732768501760 K>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59053056 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚95967795 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚368732751462400 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 845178880ßÖN2« - _wandbruntime‚3621  _runtime‚3621ƒ®Ü|v¬ ð trainer/global_step‚13599 0loss/FRt1/detection_loss‚0.47809060877304854 &loss/total_loss‚1.9432994394340068 # _timestamp‚1.78666227284202e+09 _runtime‚3621.371929716 _step‚1120 0loss/FRt1/assignment_loss‚0.5386713251890424 1loss/FRt2/assignment_loss‚0.45550173343634315 0loss/FRt2/detection_loss‚0.47103577203557273 epoch‚33 ·«ú{­ õ &loss/total_loss‚1.9432994394340068 # _timestamp‚1.78666227284202e+09 _runtime‚3621.371929716 _step‚1120 0loss/FRt1/assignment_loss‚0.5386713251890424 1loss/FRt2/assignment_loss‚0.45550173343634315 0loss/FRt2/detection_loss‚0.47103577203557273 epoch‚33 trainer/global_step‚13599 0loss/FRt1/detection_loss‚0.47809060877304854à£7]î2® - _wandbruntime‚3626  _runtime‚3626˳ý,Þ¯ Ø .particle/accuracy_2_of_2‚0.682650089263916 +particle/sensitivity‚0.6356110572814941 *particle/specificity‚0.641526460647583 -Purity/2FRt/FRt_purity‚0.5802587270736694 particle/accuracy_1_of_1‚1 'particle/f_score‚0.5558037757873535 /Purity/*FRt/event_purity‚0.4763878285884857 -Purity/1FRt/FRt_purity‚0.4784833490848541 +jet/accuracy_2_of_2‚0.22067755460739136 +validation_accuracy‚0.22067755460739136 2Purity/*FRt/event_proportion‚0.669805645942688 /particle/accuracy_1_of_2‚0.9184800982475281 Purity/0FRt/event_purity‚1 3Purity/1FRt/event_proportion‚0.6297432780265808 *jet/accuracy_1_of_2‚0.5905646085739136 7validation_average_jet_accuracy‚0.32185786962509155 % _timestamp‚1.7866622779890335e+09 _step‚1121 /Purity/2FRt/event_purity‚0.4411657750606537 *jet/accuracy_1_of_1‚0.3164791464805603 (particle/accuracy‚0.6394391655921936 _runtime‚3626.519043767 epoch‚33 *loss/val_total_loss‚2.3882801005902814 .Purity/*FRt/FRt_purity‚0.49014681577682495 3Purity/0FRt/event_proportion‚0.3301943838596344 /Purity/1FRt/event_purity‚0.4784833490848541 trainer/global_step‚13599 2Purity/2FRt/event_proportion‚0.040062315762043: Wã° Ý .particle/accuracy_2_of_2‚0.682650089263916 +particle/sensitivity‚0.6356110572814941 *particle/specificity‚0.641526460647583 -Purity/2FRt/FRt_purity‚0.5802587270736694 particle/accuracy_1_of_1‚1 'particle/f_score‚0.5558037757873535 /Purity/*FRt/event_purity‚0.4763878285884857 -Purity/1FRt/FRt_purity‚0.4784833490848541 +jet/accuracy_2_of_2‚0.22067755460739136 +validation_accuracy‚0.22067755460739136 2Purity/*FRt/event_proportion‚0.669805645942688 /particle/accuracy_1_of_2‚0.9184800982475281 Purity/0FRt/event_purity‚1 3Purity/1FRt/event_proportion‚0.6297432780265808 *jet/accuracy_1_of_2‚0.5905646085739136 7validation_average_jet_accuracy‚0.32185786962509155 % _timestamp‚1.7866622779890335e+09 _step‚1121 /Purity/2FRt/event_purity‚0.4411657750606537 *jet/accuracy_1_of_1‚0.3164791464805603 (particle/accuracy‚0.6394391655921936 _runtime‚3626.519043767 epoch‚33 *loss/val_total_loss‚2.3882801005902814 .Purity/*FRt/FRt_purity‚0.49014681577682495 3Purity/0FRt/event_proportion‚0.3301943838596344 /Purity/1FRt/event_purity‚0.4784833490848541 trainer/global_step‚13599 2Purity/2FRt/event_proportion‚0.040062315762043árN”± ‚(:‰ ‰›ùÓä‘üà gpu.0.gpu‚9.0 gpu.0.memory‚2, gpu.0.memoryAllocated‚7.0313262939453125) gpu.0.memoryAllocatedBytes‚ 6039863296 gpu.0.temp‚36.0 gpu.0.powerWatts‚211.254' gpu.0.enforcedPowerLimitWatts‚500.0 gpu.0.powerPercent‚42.2508 gpu.0.smClock‚1410 gpu.0.memoryClock‚1593$ gpu.0.correctedMemoryErrors‚1955# gpu.0.uncorrectedMemoryErrors‚0F­—² ‚(:Œ ‰›ùÓô˜ã proc.cpu.threads‚17 network.recv‚ 6.755094e+06 disk.sda.in‚33679.45703125 disk.drbd2165.in‚0! disk.nvme0c0n1.in‚ 43832.59375) disk./.usagePercent‚32.73294821881017 disk.sda1.in‚0 disk.sda2.out‚0 disk.dm-1.in‚33679.45703125 disk.dm-1.out‚ 66.6953125( disk.nvme0c0n1.out‚59105.58056640625# proc.memory.rssMB‚ 2362.08984375* proc.memory.percent‚14.417052268981934$ disk./.usageGB‚487.7445831298828 disk.sda.out‚ 66.6953125 disk.sda1.out‚0 disk.sda2.in‚0 disk.dm-0.in‚ 43832.59375# disk.dm-0.out‚59105.58056640625 disk.sda3.in‚33679.45703125) proc.memory.availableMB‚ 5139.99609375 network.sent‚ 1.7671777e+07 disk.nvme0n1.in‚ 43832.59375& disk.nvme0n1.out‚59105.58056640625 disk.drbd2165.out‚0 disk.sda3.out‚ 66.6953125$ memory_percent‚68.62795352935791 cpu‚27.217663383849725ãË/p2³ - _wandbruntime‚3638  _runtime‚3638…ÆtbÉ´ à & lr-AdamW/pg1‚0.0003561804102231469 & lr-AdamW/pg2‚0.0003561804102231469 trainer/global_step‚13649 % _timestamp‚1.7866622903063343e+09 _runtime‚3638.836352529 _step‚11229bD…ε È & lr-AdamW/pg1‚0.0003561804102231469 & lr-AdamW/pg2‚0.0003561804102231469 trainer/global_step‚13649 % _timestamp‚1.7866622903063343e+09 _runtime‚3638.836352529 _step‚1122âR„¯2¶ - _wandbruntime‚3638  _runtime‚3638T^· ºO XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚31479965 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚61478912 PCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚58387380 KDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚60 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚96390789 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 122905600 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 5368709120 IDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 4628521206 F@DevŒG¿°!iceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 125829120 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 3301635314176 IBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚86 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚322 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚45829514 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚11455990 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 3301677259264 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚45829514 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚11455906 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 3294763334078 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚262 F>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚372602373636096 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚372602373636096 PIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚84 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 3491370984960 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 5368709120 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚322 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚41348020 G>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4876 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 4654602240 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 4654602240 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 trainer/global_step‚13649 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚375903991845376 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚4287 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚50561273 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17104896 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 122905600 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚31479963 UHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚70973440 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚20643840 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚50561273 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 5494538240 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 5368709120 IBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚60 _step‚1123 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚45825227 JBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚665 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 845178880 PCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚59049984 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚328277184470528 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 IDeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4289 BDeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4289 BDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 4628521206 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 125829120 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 3301635314176 IBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚86 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚322 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚45829514 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚11455990 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 3301677259264 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚45829514 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚11455906 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 3294763334078 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚262 F>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚372602373636096 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚372602373636096 PIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚84 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 3491370984960 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 5368709120 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚322 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚41348020 G>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4876 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 4654602240 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 4654602240 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 trainer/global_step‚13649 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚375903991845376 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚4287 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚50561273 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17104896 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 122905600 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚31479963 UHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚70973440 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚20643840 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚50561273 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 5494538240 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 5368709120 IBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚60 _step‚1123 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚45825227 JBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚665 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 845178880 PCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚59049984ãJõi2¹ - _wandbruntime‚3639  _runtime‚3639nyË&º ÅM SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 122905600 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 5494538240 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 125829120 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 3301908331008 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 4517085184 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚4293 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 122306526 G@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚60 _runtime‚3639.002138723 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚50564828 EDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚42938912 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚48229376 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚370422957670400 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 3301908331008 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚209 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚328300408325632 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 3491618440704 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 3295035254758 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚262 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚372628695973888 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚50564826 B8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚322 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚322 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚322 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚45828514 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 3301866382848 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654602240 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 3301866382848 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚370422940631040 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 5494538240 >5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5264 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17104896 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 D=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚60 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚372628678868992 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚41948160 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚324808810528768 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 5368709120 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 I@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚4876 G@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚66 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 125829120 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 5368709120 trainer/global_step‚13649 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚50564826 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚322 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 3491666670080 NADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚58387428 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚96393340 EDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59053056 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚31482168 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 125829120 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚96393340 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 122905600 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚372628678868992 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚324808789884928 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 3294993906690 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 NADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚59053056 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚4293 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚20643840 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚61478912 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 5368709120 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚262 % _timestamp‚1.7866622904717593e+09 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚45832807 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 4654602240 B9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚5264 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚31482170 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 845178880 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚373717992925158 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚262 IBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚60 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17104896 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚41948160 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚50564828 H@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚665 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚11456808 NGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚64 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 831258624 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 4552982528 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚372628695973888 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚96397635 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚375930604304896 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚429 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚328300477198848 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 _step‚1124 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚11456744 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 PDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521206 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚375930545251840 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚45828514 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚42938978 SFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚68873216 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 5494538240¤pÿk» ÊM aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 3491666670080 NADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚58387428 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚96393340 EDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59053056 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚31482168 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 125829120 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚96393340 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 122905600 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚372628678868992 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚324808789884928 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 3294993906690 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 NADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚59053056 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚4293 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚20643840 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚61478912 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 5368709120 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚262 % _timestamp‚1.7866622904717593e+09 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚45832807 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 4654602240 B9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚5264 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚31482170 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 845178880 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚373717992925158 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚262 IBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚60 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17104896 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚41948160 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚50564828 H@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚665 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚11456808 NGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚64 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 831258624 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 4552982528 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚372628695973888 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚96397635 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚375930604304896 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚429 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚328300477198848 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 _step‚1124 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚11456744 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 PDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521206 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚375930545251840 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚45828514 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚42938978 SFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚68873216 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 5494538240 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 122905600 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 5494538240 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 125829120 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 3301908331008 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 4517085184 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚4293 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 122306526 G@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚60 _runtime‚3639.002138723 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚50564828 EDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚42938912 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚48229376 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚370422957670400 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 3301908331008 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚209 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚328300408325632 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 3491618440704 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 3295035254758 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚262 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚372628695973888 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚50564826 B8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚322 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚322 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚322 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚45828514 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 3301866382848 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654602240 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 3301866382848 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚370422940631040 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 5494538240 >5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5264 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17104896 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 D=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚60 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚372628678868992 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚41948160 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚324808810528768 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 5368709120 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 I@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚4876 G@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚66 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 125829120 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 5368709120 trainer/global_step‚13649 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚50564826 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚322äÖlÂÙ2¼ - _wandbruntime‚3639  _runtime‚3639ÚZÒw½ ñ % _timestamp‚1.7866622904737196e+09 _runtime‚3639.003591983 _step‚1125 /loss/FRt2/detection_loss‚0.4802010024642323 epoch‚34 0loss/FRt1/assignment_loss‚0.5080347725727093 1loss/FRt2/assignment_loss‚0.42236853001275454 0loss/FRt1/detection_loss‚0.46541341693602384 &loss/total_loss‚1.8760177219857201 trainer/global_step‚13649pät|¾ ö epoch‚34 0loss/FRt1/assignment_loss‚0.5080347725727093 1loss/FRt2/assignment_loss‚0.42236853001275454 0loss/FRt1/detection_loss‚0.46541341693602384 &loss/total_loss‚1.8760177219857201 trainer/global_step‚13649 % _timestamp‚1.7866622904737196e+09 _runtime‚3639.003591983 _step‚1125 /loss/FRt2/detection_loss‚0.4802010024642323åI[6”¿ ‚(:‰ ˜›ùÓÏÁ²á gpu.0.gpu‚0.0 gpu.0.memory‚0, gpu.0.memoryAllocated‚7.0313262939453125) gpu.0.memoryAllocatedBytes‚ 6039863296 gpu.0.temp‚35.0 gpu.0.powerWatts‚119.694' gpu.0.enforcedPowerLimitWatts‚500.0 gpu.0.powerPercent‚23.9388 gpu.0.smClock‚1410 gpu.0.memoryClock‚1593$ gpu.0.correctedMemoryErrors‚1955# gpu.0.uncorrectedMemoryErrors‚0Åôq­ƒÀ ‚(:ø ˜›ùÓŸÛÖã disk.dm-1.in‚33679.45703125 disk.dm-1.out‚ 66.8359375 disk.sda3.in‚33679.45703125 disk.sda3.out‚ 66.8359375 disk.nvme0c0n1.in‚ 43835.3125 network.sent‚ 1.7751998e+07$ disk./.usageGB‚487.7446098327637# disk.nvme0c0n1.out‚ 59112.890625$ memory_percent‚68.62688064575195' proc.memory.availableMB‚ 5140.171875# proc.memory.rssMB‚ 2362.08984375 cpu‚27.217312322304068 proc.cpu.threads‚17 network.recv‚ 6.795032e+06 disk.nvme0n1.in‚ 43835.3125 disk.drbd2165.in‚0 disk.dm-0.in‚ 43835.3125 disk.sda.in‚33679.45703125 disk.sda2.in‚0 disk.sda2.out‚0) disk./.usagePercent‚32.73295001086291! disk.nvme0n1.out‚ 59112.890625 disk.drbd2165.out‚0 disk.dm-0.out‚ 59112.890625 disk.sda.out‚ 66.8359375 disk.sda1.in‚0 disk.sda1.out‚0* proc.memory.percent‚14.417052268981934a”ß2Á - _wandbruntime‚3652  _runtime‚3652ý–³‰É Ã & lr-AdamW/pg2‚0.0003510777358455854 trainer/global_step‚13699 % _timestamp‚1.7866623034739907e+09 _runtime‚3652.005885413 _step‚1126 & lr-AdamW/pg1‚0.0003510777358455854qáÎà È trainer/global_step‚13699 % _timestamp‚1.7866623034739907e+09 _runtime‚3652.005885413 _step‚1126 & lr-AdamW/pg1‚0.0003510777358455854 & lr-AdamW/pg2‚0.0003510777358455854æ‰ø?2Ä - _wandbruntime‚3652  _runtime‚3652 v…ýÀ'Å ºO JBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚665 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 4628521206 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 3306325144686 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚377231695500800 IBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚67 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚322 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 _runtime‚3652.006023744 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚373918473420800 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚429 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚373918473420800 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚31590213 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 trainer/global_step‚13699 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 122905600 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4289 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚262 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 125829120 M@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚59049984 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 845178880 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 125829120 KDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚60 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚41945088 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚50738923 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 3503679728128 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 4552982528 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚329441855562240 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 122306526 E8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚96729252 @7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚5264 KBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚4876 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚377231754550784 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚377231695500800 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17104896 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚31590215 UHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚64681984 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚262 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚377231754550784 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 3313222080000 F?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚60 C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚4289 F>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚329441790880256 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚371706061553664 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚373918490525696 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚96733541 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚43087146 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 PCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚58387380 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚41348020 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 3503723766272 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 3306366492706 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 5494538240 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚325938131795968 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚325938111152128 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚44038144 $ _timestamp‚1.786662303474719e+09 _step‚1127 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚375012369658990 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 5368709120 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚45990329 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 4654602240 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17104896 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚45994616 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚45990329 BDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4876 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚41945088 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚373918490525696 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚43087213 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 3313264025088 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚322 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 5494538240 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 5368709120 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚61478912 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚371706044514304 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 4654602240 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 3313222080000 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 IDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚377231695500800 IBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚67 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚322 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 _runtime‚3652.006023744 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚373918473420800 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚429 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚373918473420800 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚31590213 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 trainer/global_step‚13699 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 122905600 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4289 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚262 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 125829120 M@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚59049984 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 845178880 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 125829120 KDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚60 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚41945088 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚50738923 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 3503679728128 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 4552982528 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚329441855562240 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 122306526 E8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚96729252 @7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚5264 KBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚4876 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚377231754550784 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚377231695500800 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17104896 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚31590215 UHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚64681984 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚262 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚377231754550784 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 3313222080000 F?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚60 C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚4289 F>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚329441790880256 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚371706061553664 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚373918490525696 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚96733541 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚43087146 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 PCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚58387380 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚41348020 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 3503723766272 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 3306366492706 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 5494538240 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚325938131795968 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚325938111152128 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚44038144 $ _timestamp‚1.786662303474719e+09 _step‚1127 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚375012369658990 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 5368709120 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚45990329 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 4654602240 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17104896 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚45994616 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚45990329 BDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4876 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚41945088çý¦J2Ç - _wandbruntime‚3652  _runtime‚3652{‚ƒåKÈ ÅM YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 3313495096832 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚377258307960320 BDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚43090120 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚96736092 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚4293 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚377258307960320 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚31592420 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 845178880 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚20643840 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚50742478 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚50742476 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 3313453148672 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚322 IBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚60 _runtime‚3652.183476094 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚41348068 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚96740387 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚45993616 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654602240 H@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚665 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 831258624 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 122306526 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚373944812863488 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚322 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚46132224 NADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚58387428 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚375038786025138 EDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521206 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 5368709120 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 122905600 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 4552982528 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚325961179168768 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 5494538240 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚262 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 125829120 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚322 >5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5264 NGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚77 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚371732247347200 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚4295 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚41948160 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚429 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚11497779 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚61478912 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚45993616 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚43090199 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚209 DDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59053056 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚373944812863488 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 PDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 _step‚1128 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 5368709120 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚373944795758592 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚11497702 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚4293 I@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚4876 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚223 SFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚66776064 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚322 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 4517085184 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 125829120 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚31592418 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚373944795758592 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 4552982528 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 3313495096832 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚41948160 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚96740387 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚377258248907264 G@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚79 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚371732230307840 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 125829120¡Ù %Ð&É ÊM LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚4293 I@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚4876 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚223 SFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚66776064 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚322 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 4517085184 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 125829120 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚31592418 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚373944795758592 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 4552982528 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 3313495096832 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚41948160 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚96740387 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚377258248907264 G@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚79 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚371732230307840 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 125829120 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 3313495096832 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚377258307960320 BDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚43090120 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚96736092 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚4293 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚377258307960320 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚31592420 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 845178880 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚20643840 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚50742478 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚50742476 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 3313453148672 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚322 IBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚60 _runtime‚3652.183476094 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚41348068 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚96740387 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚45993616 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654602240 H@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚665 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 831258624 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 122306526 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚373944812863488 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚322 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚46132224 NADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚58387428 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚375038786025138 EDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521206 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 5368709120 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 122905600 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 4552982528 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚325961179168768 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 5494538240 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚262 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 125829120 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚322 >5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5264 NGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚77 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚371732247347200 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚4295 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚41948160 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚429 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚11497779 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚61478912 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚45993616 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚43090199 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚209 DDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59053056 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚373944812863488 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 PDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 _step‚1128 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 5368709120 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚373944795758592 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚11497702è{ê42Ê - _wandbruntime‚3652  _runtime‚3652ë8.qË ë 0loss/FRt2/assignment_loss‚0.4832620199117985 /loss/FRt1/detection_loss‚0.5331225582840483 %loss/total_loss‚2.081850255163607 epoch‚34 _step‚1129 /loss/FRt1/assignment_loss‚0.564682245436745 .loss/FRt2/detection_loss‚0.500783431531015 trainer/global_step‚13699 $ _timestamp‚1.786662303654037e+09 _runtime‚3652.184032673Äh¤˜vÌ ð /loss/FRt1/assignment_loss‚0.564682245436745 .loss/FRt2/detection_loss‚0.500783431531015 trainer/global_step‚13699 $ _timestamp‚1.786662303654037e+09 _runtime‚3652.184032673 0loss/FRt2/assignment_loss‚0.4832620199117985 /loss/FRt1/detection_loss‚0.5331225582840483 %loss/total_loss‚2.081850255163607 epoch‚34 _step‚1129é#bã@¡Í ‚(:– §›ùÓÐúƒá gpu.0.gpu‚61.0 gpu.0.memory‚11, gpu.0.memoryAllocated‚7.0313262939453125) gpu.0.memoryAllocatedBytes‚ 6039863296 gpu.0.temp‚37.0 gpu.0.powerWatts‚188.847' gpu.0.enforcedPowerLimitWatts‚500.0) gpu.0.powerPercent‚37.769400000000005 gpu.0.smClock‚1410 gpu.0.memoryClock‚1593$ gpu.0.correctedMemoryErrors‚1955# gpu.0.uncorrectedMemoryErrors‚0דa@•Î ‚(:Š §›ùÓ½¹Üâ disk.drbd2165.out‚0$ memory_percent‚68.62704753875732 proc.cpu.threads‚17 network.recv‚ 6.827798e+06) disk./.usagePercent‚32.73295615504374 disk.sda3.in‚33679.45703125 disk.sda1.out‚0 disk.sda2.in‚0 disk.nvme0n1.in‚ 43835.3125" disk.dm-0.out‚59158.1357421875# proc.memory.rssMB‚ 2362.08984375' disk.nvme0c0n1.out‚59158.1357421875 disk.sda.in‚33679.45703125 disk.sda.out‚ 67.34765625 disk.dm-1.out‚ 67.34765625 cpu‚27.216986167945496 disk.nvme0c0n1.in‚ 43835.3125% disk.nvme0n1.out‚59158.1357421875 disk.dm-1.in‚33679.45703125 disk.drbd2165.in‚0 disk.dm-0.in‚ 43835.3125) proc.memory.availableMB‚ 5140.14453125* proc.memory.percent‚14.417052268981934 network.sent‚ 1.7827014e+07% disk./.usageGB‚487.74470138549805 disk.sda3.out‚ 67.34765625 disk.sda1.in‚0 disk.sda2.out‚0# 2Ö2Ï - _wandbruntime‚3664  _runtime‚3664…óºÉÐ Ã & lr-AdamW/pg1‚0.0003460006834996037 & lr-AdamW/pg2‚0.0003460006834996037 trainer/global_step‚13749 % _timestamp‚1.7866623159225278e+09 _runtime‚3664.454912039 _step‚1130%DK–ÎÑ È & lr-AdamW/pg1‚0.0003460006834996037 & lr-AdamW/pg2‚0.0003460006834996037 trainer/global_step‚13749 % _timestamp‚1.7866623159225278e+09 _runtime‚3664.454912039 _step‚1130êÜ#Á02Ò - _wandbruntime‚3664  _runtime‚3664‰0å|¿'Ó ¹O TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚31700463 @7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚5264 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 3317928303314 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚375234607415296 PIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚74 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚262 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚4289 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚46155431 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 F?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚60 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚375234607415296 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 125829120 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚322 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚378559458206208 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚4287 KBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚4876 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚209 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚61478912 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚46159718 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17104896 IDeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4289 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚322 PCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚59049984 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚50916573 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚11537541 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚20643840 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚41348020 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 122306526 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 125829120 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚322 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚262 IBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚60 M@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚59049984 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 $ _timestamp‚1.786662315923225e+09 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 3515964231168 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚46155431 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 845178880 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 4628521206 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 4517085184 KDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚60 PCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚58387380 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 3317886955294 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚375234590310400 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚31700465 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚11537467 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 5494538240 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚50916575 JBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚665 IBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚76 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚376333279533778 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 3324808845824 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚327090500435968 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚50916575 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 4654602240 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚97076293 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 _runtime‚ 3664.45501319 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚46159718 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚43238006 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚4287 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17104896 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 3516012463616 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 5494538240 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 5368709120 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚375234590310400 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 122905600 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 831258624 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚373015351230464 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 5494538240 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚262 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 3324808845824 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚378559458206208 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚378559399156224 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚373015334191104 IDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚376333221146398 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 5368709120 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 125829120 trainer/global_step‚13749 _step‚1131 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 5368709120 F>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 G>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4876 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 4552982528 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 122905600 D;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚5264 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚330606512899584 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 4654602240 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚482324481LÿžLÔ ¾O PCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚58387380 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 3317886955294 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚375234590310400 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚31700465 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚11537467 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 5494538240 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚50916575 JBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚665 IBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚76 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚376333279533778 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 3324808845824 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚327090500435968 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚50916575 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 4654602240 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚97076293 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 _runtime‚ 3664.45501319 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚46159718 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚43238006 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚4287 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17104896 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 3516012463616 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 5494538240 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 5368709120 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚375234590310400 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 122905600 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 831258624 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚373015351230464 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 5494538240 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚262 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 3324808845824 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚378559458206208 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚378559399156224 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚373015334191104 IDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚376333221146398 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 5368709120 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 125829120 trainer/global_step‚13749 _step‚1131 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 5368709120 F>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 G>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4876 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 4552982528 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 122905600 D;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚5264 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚330606512899584 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 4654602240 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚48232448 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚31700463 @7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚5264 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 3317928303314 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚375234607415296 PIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚74 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚262 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚4289 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚46155431 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 F?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚60 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚375234607415296 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 125829120 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚322 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚378559458206208 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚4287 KBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚4876 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚209 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚61478912 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚46159718 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17104896 IDeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4289 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚322 PCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚59049984 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚50916573 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚11537541 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚20643840 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚41348020 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 122306526 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 125829120 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚322 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚262 IBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚60 M@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚59049984 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 $ _timestamp‚1.786662315923225e+09 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 3515964231168 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚46155431 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 845178880 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 4628521206 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 4517085184 KDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚60ë'â«*2Õ - _wandbruntime‚3664  _runtime‚3664„HË&Ö ÅM REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚41948160 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚46163011 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 831258624 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 122306526 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 125829120 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚378586011615744 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚375260912648192 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 3325081862656 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚4293 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚330629805980160 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚378585952562688 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚223 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚209 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚61478912 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 3325081862656 EDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚43240968 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚373041519984640 K>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59053056 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 5368709120 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 5368709120 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚11538300 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 5494538240 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚375260912648192 H@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚665 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚262 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚330629739204096 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 trainer/global_step‚13749 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 125829120 E6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚322 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17104896 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 4552982528 B9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚5264 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚50920126 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚322 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚46163011 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 4517085184 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 3318117527906 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚97083139 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚50920128 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚376359695899974 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚376359637512546 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚50920126 NADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚59053056 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521206 _runtime‚3664.616623633 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 122905600 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654602240 SFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚66776064 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 3318158875974 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚262 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚46132224 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚11538367 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚322 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚97078844 >5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5264 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17104896 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚322 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚262 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚50920128 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚46158718 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚97078844 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚43241037 NADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚58387428 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 5494538240 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚97083139 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 NGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚67 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 5368709120 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 125829120 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚375260929753088 DDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 _step‚1132 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚378586011615744 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚31702670 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚31702668 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚41348068 D=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚60 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 4654602240 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚375260929753088 BDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 PDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 _step‚1132 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚378586011615744 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚31702670 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚31702668 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚41348068 D=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚60 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 4654602240 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚375260929753088 BDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 PDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚43240968 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚373041519984640 K>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59053056 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 5368709120 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 5368709120 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚11538300 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 5494538240 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚375260912648192 H@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚665 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚262 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚330629739204096 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 trainer/global_step‚13749 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 125829120 E6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚322 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17104896 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 4552982528 B9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚5264 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚50920126 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚322 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚46163011 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 4517085184 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 3318117527906 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚97083139 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚50920128 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚376359695899974 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚376359637512546 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚50920126 NADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚59053056 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521206 _runtime‚3664.616623633 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 122905600 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654602240 SFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚66776064 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 3318158875974 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚262 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚46132224 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚11538367 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚322 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚97078844 >5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5264 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17104896 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚322 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚262 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚50920128 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚46158718 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚97078844 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚43241037 NADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚58387428 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 5494538240 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚97083139 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 NGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚67 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0ì¥çT2Ø - _wandbruntime‚3664  _runtime‚3664À›üuÙ ï /loss/FRt1/detection_loss‚0.5060040620460079 /loss/FRt2/detection_loss‚0.4485484616684728 epoch‚34 trainer/global_step‚13749 % _timestamp‚1.7866623160874743e+09 _runtime‚3664.617384039 0loss/FRt1/assignment_loss‚0.5796350816167728 0loss/FRt2/assignment_loss‚0.4594808813421089 &loss/total_loss‚1.9936684866733625 _step‚1133jMÍÙzÚ ô epoch‚34 trainer/global_step‚13749 % _timestamp‚1.7866623160874743e+09 _runtime‚3664.617384039 0loss/FRt1/assignment_loss‚0.5796350816167728 0loss/FRt2/assignment_loss‚0.4594808813421089 &loss/total_loss‚1.9936684866733625 _step‚1133 /loss/FRt1/detection_loss‚0.5060040620460079 /loss/FRt2/detection_loss‚0.4485484616684728ílÿçjŸÛ ‚(:” ¶›ùÓÆá á gpu.0.gpu‚0.0 gpu.0.memory‚0, gpu.0.memoryAllocated‚7.0313262939453125) gpu.0.memoryAllocatedBytes‚ 6039863296 gpu.0.temp‚36.0 gpu.0.powerWatts‚218.269' gpu.0.enforcedPowerLimitWatts‚500.0) gpu.0.powerPercent‚43.653800000000004 gpu.0.smClock‚1410 gpu.0.memoryClock‚1593$ gpu.0.correctedMemoryErrors‚1955# gpu.0.uncorrectedMemoryErrors‚0‡x<,†Ü ‚(:û ¶›ùÓçÖúã network.recv‚ 6.85978e+06) disk./.usagePercent‚32.73292108201154 disk.sda1.in‚0 disk.sda1.out‚0# proc.memory.rssMB‚ 2362.08984375 cpu‚27.21662945724732 proc.cpu.threads‚17% disk./.usageGB‚487.74417877197266 disk.sda3.in‚33679.45703125 disk.sda3.out‚ 67.55078125 disk.sda2.in‚0 disk.sda2.out‚0 disk.nvme0n1.in‚ 43836.96875 disk.dm-1.in‚33679.45703125 disk.drbd2165.in‚0 network.sent‚ 1.7901966e+07" disk.nvme0c0n1.out‚ 59166.28125 disk.drbd2165.out‚0 disk.dm-0.in‚ 43836.96875 disk.dm-0.out‚ 59166.28125$ memory_percent‚68.62730979919434( proc.memory.availableMB‚ 5140.1015625* proc.memory.percent‚14.417052268981934! disk.nvme0c0n1.in‚ 43836.96875 disk.sda.in‚33679.45703125 disk.sda.out‚ 67.55078125 disk.nvme0n1.out‚ 59166.28125 disk.dm-1.out‚ 67.55078125–!-;2Ý - _wandbruntime‚3677  _runtime‚3677~¦ìáÉÞ Ã _step‚1134 & lr-AdamW/pg1‚0.0003409495792747866 & lr-AdamW/pg2‚0.0003409495792747866 trainer/global_step‚13799 % _timestamp‚1.7866623289046955e+09 _runtime‚3677.434822973º™ÎÎß È _step‚1134 & lr-AdamW/pg1‚0.0003409495792747866 & lr-AdamW/pg2‚0.0003409495792747866 trainer/global_step‚13799 % _timestamp‚1.7866623289046955e+09 _runtime‚3677.434822973îÒ·Ð72à - _wandbruntime‚3677  _runtime‚3677”sáá »O WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 125829120 C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚4289 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚379887102811648 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚377654131021186 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚377654072633806 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 4552982528 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 4552982528 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚20643840 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 trainer/global_step‚13799 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚223 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚429 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚31810713 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚331771109699072 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚322 E8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚97414756 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 3336395611648 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚4287 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚51094223 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚50329600 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 5494538240 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚262 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 5494538240 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 3336437556736 IDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4876 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚262 _step‚1135 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 125829120 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 122905600 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4289 D;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚5264 JBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚665 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 3329448765902 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 4628521206 _runtime‚3677.439917341 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚51094223 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚41945088 UHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚70973440 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 F?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚60 F>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚376550707200000 PCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚59049984 KBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚4876 PIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚67 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚379887102811648 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 5368709120 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚11577899 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚209 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 3528261266944 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 125829120 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚322 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚376550724304896 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚46324820 IBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚60 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚46324820 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 3528311596544 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚379887161861632 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚374324640907264 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0__“Æ'â ÀO YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚379887161861632 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚374324640907264 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 125829120 C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚4289 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚379887102811648 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚377654131021186 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚377654072633806 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 4552982528 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 4552982528 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚20643840 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 trainer/global_step‚13799 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚223 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚429 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚31810713 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚331771109699072 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚322 E8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚97414756 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 3336395611648 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚4287 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚51094223 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚50329600 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 5494538240 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚262 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 5494538240 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 3336437556736 IDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4876 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚262 _step‚1135 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 125829120 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 122905600 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4289 D;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚5264 JBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚665 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 3329448765902 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 4628521206 _runtime‚3677.439917341 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚51094223 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚41945088 UHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚70973440 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 F?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚60 F>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚376550707200000 PCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚59049984 KBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚4876 PIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚67 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚379887102811648 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 5368709120 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚11577899 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚209 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 3528261266944 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 125829120 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚322 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚376550724304896 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚46324820 IBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚60 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚46324820 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 3528311596544 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0ï¦ÀI}2ã - _wandbruntime‚3677  _runtime‚3677Ž°ÏË&ä ÅM T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚379913715271168 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚376577046642688 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚41948160 E6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚322 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 5494538240 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚20643840 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 3528504364544 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚377680547387382 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 5494538240 B9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚5264 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚46328113 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 DDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654602240 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚97421596 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚429 B6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚322 % _timestamp‚1.7866623290802457e+09 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚376577029537792 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚379913715271168 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚97425891 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 5494538240 NADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚59053056 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚223 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚331794400169472 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 3329720686582 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚379913656218112 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 3336626680320 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚11578783 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚328265916448768 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚322 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚51097778 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚4293 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 3528558885376 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚54520832 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 5368709120 IBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚60 D=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚60 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚376577029537792 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 125829120 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 125829120 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚262 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚61478912 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 4552982528 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 3336668628480 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚31812920 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚97425891 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚46328113 EDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 _step‚1136 >5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5264 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17104896 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 4517085184 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚262 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 3329679338514 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 122306526 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚4295 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚46323820 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚46323820 I@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚4876 G@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚87 NGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚85 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 H@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚665 SFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚75164672 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 125829120 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 3336626680320 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚4293 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚41948160 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 122905600 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚11578698 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 831258624 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚322 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚51097776 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521206 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 trainer/global_step‚13799 K>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59053056 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17104896 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚43391616 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 5368709120 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚97421596 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 845178880 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚377680488999954 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 5368709120 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 4654602240 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 _runtime‚3677.610649958 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚331794475334144 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚374350826700800I^Pqå ÊM ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚54520832 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 5368709120 IBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚60 D=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚60 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚376577029537792 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 125829120 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 125829120 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚262 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚61478912 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 4552982528 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 3336668628480 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚31812920 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚97425891 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚46328113 EDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 _step‚1136 >5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5264 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17104896 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 4517085184 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚262 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 3329679338514 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 122306526 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚4295 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚46323820 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚46323820 I@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚4876 G@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚87 NGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚85 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 H@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚665 SFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚75164672 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 125829120 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 3336626680320 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚4293 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚41948160 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 122905600 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚11578698 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 831258624 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚322 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚51097776 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521206 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 trainer/global_step‚13799 K>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59053056 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17104896 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚43391616 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 5368709120 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚97421596 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 845178880 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚377680488999954 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 5368709120 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 4654602240 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 _runtime‚3677.610649958 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚331794475334144 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚374350826700800 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚379913715271168 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚376577046642688 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚41948160 E6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚322 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 5494538240 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚20643840 ]KDeviceStatsMonitor.on_train_batch_end/i/<ôÙ_nactive_split_bytes.small_pool.freed‚ 3528504364544 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚377680547387382 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 5494538240 B9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚5264 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚46328113 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 DDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654602240 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚97421596 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚429 B6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚322 % _timestamp‚1.7866623290802457e+09 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚376577029537792 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚379913715271168 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚97425891 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 5494538240 NADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚59053056 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚223 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚331794400169472 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 3329720686582 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚379913656218112 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 3336626680320 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚11578783 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚328265916448768 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚322 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚51097778 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚4293 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 3528558885376ð:Yâ¢2æ - _wandbruntime‚3677  _runtime‚3677_ûÒHtç î % _timestamp‚1.7866623290815146e+09 0loss/FRt1/assignment_loss‚0.6298424366436389 0loss/FRt2/assignment_loss‚0.4332330619277989 %loss/total_loss‚2.028554819944837 epoch‚34 _runtime‚3677.611379196 _step‚1137 /loss/FRt1/detection_loss‚0.4863471308676575 /loss/FRt2/detection_loss‚0.4791321905057418 trainer/global_step‚13799<®zÒyè ó trainer/global_step‚13799 % _timestamp‚1.7866623290815146e+09 0loss/FRt1/assignment_loss‚0.6298424366436389 0loss/FRt2/assignment_loss‚0.4332330619277989 %loss/total_loss‚2.028554819944837 epoch‚34 _runtime‚3677.611379196 _step‚1137 /loss/FRt1/detection_loss‚0.4863471308676575 /loss/FRt2/detection_loss‚0.4791321905057418ñ3,]…”é ‚(:‰ Å›ùÓÓˆ˜á gpu.0.gpu‚0.0 gpu.0.memory‚0, gpu.0.memoryAllocated‚7.0313262939453125) gpu.0.memoryAllocatedBytes‚ 6039863296 gpu.0.temp‚34.0 gpu.0.powerWatts‚154.492' gpu.0.enforcedPowerLimitWatts‚500.0 gpu.0.powerPercent‚30.8984 gpu.0.smClock‚1410 gpu.0.memoryClock‚1593$ gpu.0.correctedMemoryErrors‚1955# gpu.0.uncorrectedMemoryErrors‚0иWX™ê ‚(:Ž Å›ùÓéÑšã network.recv‚ 6.886585e+06# disk.dm-0.out‚59210.95458984375 disk.sda.in‚33679.45703125 disk.sda1.in‚0 disk.drbd2165.in‚0 disk.sda3.out‚ 67.86328125! disk.nvme0c0n1.in‚ 43836.96875 disk.sda.out‚ 67.86328125 disk.sda1.out‚0 disk.sda2.out‚0$ memory_percent‚68.62916946411133 cpu‚27.216305389140686 network.sent‚ 1.7971545e+07 disk.nvme0n1.in‚ 43836.96875& disk.nvme0n1.out‚59210.95458984375 disk.dm-1.in‚33679.45703125 disk.sda3.in‚33679.45703125 disk.sda2.in‚0* proc.memory.percent‚14.417052268981934 proc.cpu.threads‚17) disk./.usagePercent‚32.73292261805675% disk./.usageGB‚487.74420166015625 disk.dm-1.out‚ 67.86328125 disk.drbd2165.out‚0 disk.dm-0.in‚ 43836.96875( disk.nvme0c0n1.out‚59210.95458984375' proc.memory.availableMB‚ 5139.796875# proc.memory.rssMB‚ 2362.08984375¶¿Âî2ë - _wandbruntime‚3690  _runtime‚3690&&áoËì Å ' lr-AdamW/pg1‚0.00033592474759411913 ' lr-AdamW/pg2‚0.00033592474759411913 trainer/global_step‚13849 % _timestamp‚1.7866623415855973e+09 _runtime‚3690.115598792 _step‚1138¿ìC?Ðí Ê ' lr-AdamW/pg1‚0.00033592474759411913 ' lr-AdamW/pg2‚0.00033592474759411913 trainer/global_step‚13849 % _timestamp‚1.7866623415855973e+09 _runtime‚3690.115598792 _step‚1138ò*&i12î - _wandbruntime‚3690  _runtime‚3690Þl>ÖÁ'ï »O IBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚68 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚377866824089600 @7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚5264 UHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚66779136 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 3341051924530 IBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚60 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚46485635 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚31920965 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚262 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚322 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 5368709120 IDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚381214806467072 PIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚66 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚329395237715968 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚46489922 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚377866841194496 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚223 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚332935828638208 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚375633913544704 _step‚1139 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚378974924121214 G>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4876 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 125829120 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚429 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚31920963 E8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚97757508 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚51271873 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 3347982377472 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚262 KDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚60 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚43539320 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚11618355 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 845178880 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚322 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 4552982528 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17104896 IDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 5368709120 trainer/global_step‚13849 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚329395217072128 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 BDeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4289 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚381214865517056 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚377866841194496 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 3540590922240 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 4628521206 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 122306526 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 KBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚4876 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚51271873 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚46485635 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚61478912 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 3341010576510 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 3348024322560 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 122905600 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚209 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚378974982508594 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚4289 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚332935761859072 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 4517085184 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚322 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 831258624 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 125829120 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚51271875 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚20643840 F?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚60 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚4287 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚46489922 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 3540544786944 M@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚59049984 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚41945088 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚41945088 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚322 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚262 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚381214865517056 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 122905600 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚46135296 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚43539252 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚41348020 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚51271875 JBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚665¦»Æ³Æ'ð ÀO TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚11618289 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 5494538240 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚97761797 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚381214806467072 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 4654602240 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 4654602240 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚375633930584064 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 125829120 % _timestamp‚1.7866623415865715e+09 _runtime‚3690.119877569 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚377866824089600 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 4552982528 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4289 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚381214865517056 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚377866841194496 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 3540590922240 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 4628521206 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 122306526 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 KBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚4876 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚51271873 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚46485635 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚61478912 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 3341010576510 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 3348024322560 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 122905600 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚209 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚378974982508594 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚4289 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚332935761859072 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 4517085184 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚322 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 831258624 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 125829120 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚51271875 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚20643840 F?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚60 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚4287 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚46489922 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 3540544786944 M@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚59049984 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚41945088 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚41945088 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚322 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚262 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚381214865517056 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 122905600 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚46135296 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚43539252 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚41348020 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚51271875 JBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚665 IBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚68 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚377866824089600 @7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚5264 UHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚66779136 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 3341051924530 IBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚60 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚46485635 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚31920965 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚262 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚322 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 5368709120 IDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚381214806467072 PIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚66 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚329395237715968 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚46489922 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚377866841194496 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚223 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚332935828638208 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚375633913544704 _step‚1139 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚378974924121214 G>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4876 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 125829120 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚429 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚31920963 E8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚97757508 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚51271873 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 3347982377472 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚262 KDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚60 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚43539320 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚11618355 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 845178880 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚322 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 4552982528 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17104896 IDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 5368709120 trainer/global_step‚13849 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚329395217072128 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 B6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚322 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 I@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚4876 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 NADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚58387428 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚381241359873536 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚97768643 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚375660116377600 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 125829120 IBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚60 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚332959056359936 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚61478912 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 5368709120 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚4293 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 5368709120 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚97768643 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚4295 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚97764348 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚377893146427392 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚20643840 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚31923170 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 831258624 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 125829120 K>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59053056 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚43542259 _step‚1140 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚381241418926592 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚377893146427392 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 122905600 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 3540833852928 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚379001340487362 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚375660099338240 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚262 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚262 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚51275428 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚4293 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚41948160 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚209 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 5494538240 % _timestamp‚1.7866623417489545e+09 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚51275426 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚377893163532288 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 3348255394304 G@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚71 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚262 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚46493215 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 4654602240 B9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚5264 BDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521206 H@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚665 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚46488922 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 3540791915008 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚41348068 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 3348213446144 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 4517085184 SEDeviceStatsMonitor.on_train_batch_end/requested_byLíetes.small_pool.peak‚ 122306526 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚377893163532288 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚381241418926592 NADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚59053056 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 3341282497190 G@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚60 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚46488922 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 4552982528 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 122905600 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 845178880 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚322 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚322 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17104896 NGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚69 PDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654602240 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚11619160 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 5494538240 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 D5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5264 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚46493215 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 4552982528 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚97764348 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚41937920 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 3348255394304 EDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521206 H@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚665 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚46488922 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 3540791915008 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚41348068 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 3348213446144 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 4517085184 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 122306526 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚377893163532288 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚381241418926592 NADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚59053056 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 3341282497190 G@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚60 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚46488922 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 4552982528 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 122905600 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 845178880 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚322 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚322 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17104896 NGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚69 PDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654602240 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚11619160 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 5494538240 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 D5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5264 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚46493215 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 4552982528 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚97764348 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚41937920 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 3348255394304 E6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚322 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 I@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚4876 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 NADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚58387428 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚381241359873536 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚97768643 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚375660116377600 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 125829120 IBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚60 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚332959056359936 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚61478912 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 5368709120 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚4293 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 5368709120 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚97768643 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚4295 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚97764348 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚377893146427392 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚20643840 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚31923170 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 831258624 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 125829120 K>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59053056 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚43542259 _step‚1140 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚381241418926592 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚377893146427392 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 122905600 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 3540833852928 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚379001340487362 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚375660099338240 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚262 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚262 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚51275428 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚4293 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚41948160 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚209 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 5494538240 % _timestamp‚1.7866623417489545e+09 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚51275426 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚377893163532288ô.ÏðÍ2ô - _wandbruntime‚3690  _runtime‚3690߶?§tõ î _runtime‚3690.279807542 _step‚1141 /loss/FRt2/assignment_loss‚0.439352107950469 0loss/FRt1/detection_loss‚0.46299888405437667 .loss/FRt2/detection_loss‚0.475697895345171 &loss/total_loss‚1.8948882580989086 epoch‚34 0loss/FRt1/assignment_loss‚0.5168393707488921 trainer/global_step‚13849 % _timestamp‚1.7866623417499914e+099ÔõRyö ó % _timestamp‚1.7866623417499914e+09 _runtime‚3690.279807542 _step‚1141 /loss/FRt2/assignment_loss‚0.439352107950469 0loss/FRt1/detection_loss‚0.46299888405437667 .loss/FRt2/detection_loss‚0.475697895345171 &loss/total_loss‚1.8948882580989086 epoch‚34 0loss/FRt1/assignment_loss‚0.5168393707488921 trainer/global_step‚13849õ¬©Þ2÷ - _wandbruntime‚3702  _runtime‚3702¯ì%Éø à & lr-AdamW/pg1‚0.0003309265111931515 & lr-AdamW/pg2‚0.0003309265111931515 trainer/global_step‚13899 % _timestamp‚1.7866623541896935e+09 _runtime‚3702.719752368 _step‚1142r¢Ë˜Îù È _step‚1142 & lr-AdamW/pg1‚0.0003309265111931515 & lr-AdamW/pg2‚0.0003309265111931515 trainer/global_step‚13899 % _timestamp‚1.7866623541896935e+09 _runtime‚3702.719752368ö.då 2ú - _wandbruntime‚3702  _runtime‚3702  |Á'û »O L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚46650737 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 4654602240 IDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4876 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚11658831 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚322 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 3352572387118 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚382542510122496 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 4654602240 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 3359611088384 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4289 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 3552837562368 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚4289 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚61478912 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚262 _step‚1143 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 3552883697664 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 125829120 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 3359611088384 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚46655024 PIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚71 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚380295775608622 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 125829120 D;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚5264 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚43690046 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚379182940979200 IBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚73 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚334100490053632 UHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚66779136 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 125829120 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚51449523 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 122905600 JBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚665 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚32031215 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 4628521206 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚209 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 5494538240 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 5494538240 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚322 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚4287 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚41945088 BDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚46135296 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚262 PCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚59049984 IDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚379182940979200 IBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚73 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚334100490053632 UHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚66779136 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 125829120 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚51449523 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 122905600 JBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚665 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚32031215 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 4628521206 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚209 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 5494538240 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 5494538240 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚322 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚4287 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚41945088 BDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚46135296 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚262 PCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚59049984 IDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4876 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚11658831 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚322 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 3352572387118 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚382542510122496 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 4654602240 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 3359611088384 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4289 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 3552837562368÷Õ¥º2ý - _wandbruntime‚3702  _runtime‚3702L¿‡ÀÊ&þ ÄM H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚262 trainer/global_step‚13899 H@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚665 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚43693002 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚4293 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚330570653728768 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚46654024 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚41948160 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚4293 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚51453078 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚43693072 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 K>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59053056 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚429 NGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚68 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 125829120 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚98107100 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚32033418 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521206 D=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚60 >5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5264 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚46658317 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚380322250362198 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚41348068 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚379209263316992 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654602240 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚41937920 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 3352802959730 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 122306526 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚98111395 SFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚62581760 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 5368709120 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚98111395 PDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 3359842160128 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚46654024 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚334123718815744 D6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚322 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚32033420 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚223 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 4552982528 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚20643840 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 _runtime‚3702.896936408 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 4517085184 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚376969389015040 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚379209263316992 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 5494538240 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17104896 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 3352844307798 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 4654602240 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚209 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚98107100 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚41948160 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚51453076 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚11659584 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 3553085730816 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 G@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚60 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 5494538240 E6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚322 _step‚1144 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 3359800211968 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚61478912 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚322 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚46658317 E6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚322 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚32033420 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚223 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 4552982528 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚20643840 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 _runtime‚3702.896936408 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 4517085184 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚376969389015040 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚379209263316992 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 5494538240 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17104896 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 3352844307798 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 4654602240 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚209 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚98107100 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚41948160 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚51453076 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚11659584 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 3553085730816 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 G@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚60 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 5494538240 E6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚322 _step‚1144 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 3359800211968 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚61478912 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚322 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚46658317 EDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚43693002 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚4293 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚330570653728768 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚46654024 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚41948160 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚4293 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚51453078 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚43693072 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 K>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59053056 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚429 NGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚68 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 125829120 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚98107100 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚32033418 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521206 D=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚60 >5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5264 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚46658317 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚380322250362198 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚41348068 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚379209263316992 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654602240 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚41937920 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 3352802959730 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 122306526 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚98111395 SFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚62581760 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 5368709120 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚98111395 PDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 3359842160128 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚46654024 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚334123718815744 DDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚383870213777920 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 5494538240 @7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚5264 _step‚1147 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17104896 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 4552982528 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚378252509937664 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚322 KBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚4876 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚378252492898304 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚380499074973696 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 3364175545746 F?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚60 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚41945088 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4289 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚4287 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚380499074973696 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚43841056 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 3565132551168 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 5368709120 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 125829120 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚262 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚46135296 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 4628521206 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 125829120 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚383870272827904 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚383870213777920 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚41945088 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 122905600 IBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚72 F>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚98447301 PIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚70 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚11699593 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 5368709120 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚46815839 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 3371155909120 PCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚59049984 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚335265153682432 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚381616685483410 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚383870272827904 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 5494538240 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚429 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 122905600 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚43841128 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚51627173 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚331699974995968 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚331699954352128 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚262 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 122306526 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 5494538240 KDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚60 C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚4289 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚51627173 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚381616627096030 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚46820126 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 5368709120 trainer/global_step‚13949 E8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚98443012 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚380499057868800 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 3371155909120 JBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚665 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 845178880 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 4654602240 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17104896 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚P–)ù› 32141463 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚209 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 4654602240 IDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4876 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 3371197854208 UHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚66779136 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚61478912 _runtime‚ 3715.65295854 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚223 IDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚98447301 PIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚70 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚11699593 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 5368709120 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚46815839 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 3371155909120 PCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚59049984 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚335265153682432 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚381616685483410 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚383870272827904 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 5494538240 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚429 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 122905600 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚43841128 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚51627173 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚331699974995968 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚331699954352128 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚262 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 122306526 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 5494538240 KDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚60 C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚4289 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚51627173 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚381616627096030 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚46820126 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 5368709120 trainer/global_step‚13949 E8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚98443012 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚380499057868800 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 3371155909120 JBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚665 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 845178880 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 4654602240 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17104896 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚32141463 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚209 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 4654602240 IDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4876 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 3371197854208 UHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚66779136 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚61478912 _runtime‚ 3715.65295854 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚223 IDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚383870213777920 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 5494538240 @7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚5264 _step‚1147 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17104896 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 4552982528 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚378252509937664 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚322 KBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚4876 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚378252492898304 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚380499074973696 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 3364175545746 F?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚60 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚41945088 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4289 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚4287 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚380499074973696 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚43841056 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 3565132551168 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 5368709120 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 125829120 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚262 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚46135296 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 4628521206 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 125829120 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚383870272827904 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚383870213777920 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚41945088 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 122905600 IBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚72ûÿ?0–2‹!- _wandbruntime‚3715  _runtime‚3715FiXË&Œ!ÅM T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚383896826237440 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 D=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚60 B9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚5264 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚51630728 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚4293 E6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚322 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 3371428925952 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚429 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 831258624 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 5494538240 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚322 D5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5264 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 4654602240 G@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚69 SFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚66776064 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚20643840 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚331723022368768 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚322 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚262 NADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚59053056 E8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚378278695731200 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 4517085184 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚61478912 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521206 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 125829120 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚262 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚380525397311488 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17104896 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚41948160 NGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚67 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 3364406118406 G@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚60 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 4552982528 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 845178880 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 5368709120 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 125829120 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚4295 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚46132224 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚381643101849606 IBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚60 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚11700486 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚223 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚335288446607360 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654602240 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 3371386977792 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚209 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 BDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 % _timestamp‚1.7866623672835364e+09 _step‚1148 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 3364364770338 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 125829120 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚383896826237440 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17104896 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚331723001724928 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚262 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚98449852 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚380525380206592 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚11700419 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 122905600 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚98454147 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚4293 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 4552982528 K>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59053056 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚51630726 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚46819126 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚51630728 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚43844087 trainer/global_step‚13949 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 3371386977792 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚32143668 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚335288379831296 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚41948160 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 3565424238592 _runtime‚3715.813977259 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚46823419 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚381643043462178 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 5368709120 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚32143670 PDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚98454147 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚46823419 I@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚4876 NADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚58387428 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 122306526 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚380525380206592 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 3565378106368 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚41348068 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚383896767184384 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚51630726 H@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚665„Ò8€$!ÊM G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚98454147 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚4293 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 4552982528 K>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59053056 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚51630726 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚46819126 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚51630728 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚43844087 trainer/global_step‚13949 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 3371386977792 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚32143668 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚335288379831296 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚41948160 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 3565424238592 _runtime‚3715.813977259 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚46823419 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚381643043462178 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 5368709120 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚32143670 PDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚98454147 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚46823419 I@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚4876 NADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚58387428 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 122306526 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚380525380206592 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 3565378106368 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚41348068 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚383896767184384 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚51630726 H@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚665 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚383896826237440 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 D=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚60 B9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚5264 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚51630728 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚4293 E6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚322 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 3371428925952 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚429 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 831258624 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 5494538240 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚322 D5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5264 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 4654602240 G@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚69 SFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚66776064 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚20643840 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚331723022368768 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚322 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚262 NADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚59053056 E8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚378278695731200 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 4517085184 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚61478912 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521206 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 125829120 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚262 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚380525397311488 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17104896 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚41948160 NGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚67 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 3364406118406 G@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚60 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 4552982528 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 845178880 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 5368709120 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 125829120 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚4295 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚46132224 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚381643101849606 IBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚60 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚11700486 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚223 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚335288446607360 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654602240 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 3371386977792 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚209 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 BDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 % _timestamp‚1.7866623672835364e+09 _step‚1148 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 3364364770338 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 125829120 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚383896826237440 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17104896 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚331723001724928 Es‡ƒP=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚262 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚98449852 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚380525380206592 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚11700419 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 122905600 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0üc¦›I2Ž!- _wandbruntime‚3715  _runtime‚3715U Ý3r!ì /loss/FRt2/detection_loss‚0.4444629371452407 &loss/total_loss‚2.0450095443951315 % _timestamp‚1.7866623672850473e+09 _step‚1149 0loss/FRt1/assignment_loss‚0.6048385474604637 .loss/FRt2/assignment_loss‚0.47495379654434 .loss/FRt1/detection_loss‚0.520754263245087 epoch‚34 trainer/global_step‚13949 _runtime‚3715.814931963k}Ivw!ñ 0loss/FRt1/assignment_loss‚0.6048385474604637 .loss/FRt2/assignment_loss‚0.47495379654434 .loss/FRt1/detection_loss‚0.520754263245087 epoch‚34 trainer/global_step‚13949 _runtime‚3715.814931963 /loss/FRt2/detection_loss‚0.4444629371452407 &loss/total_loss‚2.0450095443951315 % _timestamp‚1.7866623672850473e+09 _step‚1149ý†G½•‘!‚(:Š ã›ùÓøå®á gpu.0.gpu‚26.0 gpu.0.memory‚6, gpu.0.memoryAllocated‚7.0313262939453125) gpu.0.memoryAllocatedBytes‚ 6039863296 gpu.0.temp‚33.0 gpu.0.powerWatts‚119.276' gpu.0.enforcedPowerLimitWatts‚500.0 gpu.0.powerPercent‚23.8552 gpu.0.smClock‚1410 gpu.0.memoryClock‚1593$ gpu.0.correctedMemoryErrors‚1955# gpu.0.uncorrectedMemoryErrors‚0Äà]“’!‚(:ˆ ã›ùÓÍźã disk.sda1.in‚0 disk.sda2.out‚0 disk.drbd2165.in‚0 disk.dm-0.in‚ 43886.4921875 network.recv‚ 6.950865e+06 disk.sda3.out‚68.3125# disk.nvme0c0n1.in‚ 43886.4921875 disk.sda.out‚68.3125 disk.sda1.out‚0& disk.nvme0n1.out‚59260.77685546875 disk.dm-1.in‚34872.39453125$ memory_percent‚68.62845420837402 disk.sda.in‚34872.39453125 disk.sda2.in‚0 disk.drbd2165.out‚0( proc.memory.availableMB‚ 5139.9140625# proc.memory.rssMB‚ 2362.08984375* proc.memory.percent‚14.417052268981934* disk./.usagePercent‚32.732916985890995! disk.nvme0n1.in‚ 43886.4921875 disk.dm-1.out‚68.3125# disk.dm-0.out‚59260.77685546875 cpu‚27.21557256650708 proc.cpu.threads‚17 network.sent‚ 1.8141568e+07$ disk./.usageGB‚487.7441177368164 disk.sda3.in‚34872.39453125( disk.nvme0c0n1.out‚59260.77685546875¬%»72“!- _wandbruntime‚3728  _runtime‚3728 '¦Ë”!Å ' lr-AdamW/pg1‚0.00032101110661106596 ' lr-AdamW/pg2‚0.00032101110661106596 trainer/global_step‚13999 % _timestamp‚1.7866623795270343e+09 _runtime‚3728.057040214 _step‚1150ˈ¢Ð•!Ê _step‚1150 ' lr-AdamW/pg1‚0.00032101110661106596 ' lr-AdamW/pg2‚0.00032101110661106596 trainer/global_step‚13999 % _timestamp‚1.7866623795270343e+09 _runtime‚3728.057040214þ0¼è2–!- _wandbruntime‚3728  _runtime‚3728¤šÁ'—!»O ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 5368709120 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚322 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚262 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚381815174758400 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 125829120 _step‚1151 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚32251713 C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚4289 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚51804823 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚381815174758400 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 831258624 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 3375696008334 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚4287 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚385197917433344 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚43992137 F?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚60 D;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚5264 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚4287 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚61478912 E8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚98785764 G>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4876 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17104896 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚385197976483328 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚98790053 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚223 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚322 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 122905600 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚382937536970818 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚32251715 M@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚59049984 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚11740493 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚20643840 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 3375737356354 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚385197976483328 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 4628521206 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚322 KDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚60 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚381815191863296 KBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚4876 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚48232448 @7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚5264 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚41945088 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚336429763000320 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 PCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚59049984 _runtime‚3728.061889921 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚46985228 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚379561799614464 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 4654602240 UHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚68876288 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 125829120 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚262 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 4552982528 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 3382742674944 PIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚69 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚336429831876608 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚379561782575104 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 5494538240 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17104896 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚51804823 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚46980941 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚429 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 % _timestamp‚1.7866623795279317e+09 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚11740424 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 4654602240 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 5494538240 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 trainer/global_step‚13999 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚46980941 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚385197917433344 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 3577440008192 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚262 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 4552982528 JBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚665 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚43992208 IBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚71 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚332852343635968 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 845178880 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚332852322992128 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 4517085184 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 3382784620032 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4289 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 IDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 PCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚58387380 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚382937478583438 IBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚60 F>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 3577488240640 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚41348020 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 122306526 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 125829120 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚322 IDeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4289 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 IDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 PCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚58387380 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚382937478583438 IBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚60 F>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 3577488240640 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚41348020 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 122306526 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 125829120 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚322 IDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚385197917433344 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚43992137 F?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚60 D;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚5264 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚4287 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚61478912 E8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚98785764 G>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4876 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17104896 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚385197976483328 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚98790053 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚223 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚322 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 122905600 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚382937536970818 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚32251715 M@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚59049984 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚11740493 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚20643840 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 3375737356354 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚385197976483328 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 4628521206 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚322 KDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚60 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚381815191863296 KBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚4876 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚48232448 @7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚5264 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚41945088 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚336429763000320 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 PCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚59049984 _runtime‚3728.061889921 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚46985228 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚379561799614464ÿÕS2™!- _wandbruntime‚3728  _runtime‚3728{½T¥S#š!ÅM E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚262 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 4552982528 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚98796899 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚429 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 G@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚60 NADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚59053056 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚41948160 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚51808376 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚262 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚46988521 % _timestamp‚1.7866623796964948e+09 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17104896 E5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5264 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 4552982528 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚379587968368640 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 5494538240 EDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 K>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59053056 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 3382973743616 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 3382973743616 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚332875391008768 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚322 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚41948160 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚385224470839808 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚4293 NGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚75 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 3375926580946 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 125829120 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 4654602240 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚98796899 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚223 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚11741278 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚61478912 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 4517085184 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚381841497096192 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 3383015691776 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚322 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚379587985408000 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚41348068 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚381841514201088 SFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚66776064 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17104896 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚43995273 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚336453053143040 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚20643840 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚46132224 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 H@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚665 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚322 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚46984228 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚382963894949586 _step‚1152 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 122905600 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 IBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚60 _runtime‚3728.226880091 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 831258624 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 5368709120 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚51808378 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚385224529892864 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 3577728910336 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 3577682778112 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 3375967929014 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚385224529892864 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚381841497096192 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654602240 G@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚77 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚43995196 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚11741353 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚46988521 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚382963953337014 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 5368709120 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚262 DDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521206 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 3383015691776 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚209 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2~’ÅäÐ&›!ÊM NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚46988521 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚382963953337014 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 5368709120 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚262 DDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521206 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 3383015691776 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚209 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚262 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 4552982528 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚98796899 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚429 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 G@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚60 NADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚59053056 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚41948160 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚51808376 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚262 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚46988521 % _timestamp‚1.7866623796964948e+09 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17104896 E5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5264 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 4552982528 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚379587968368640 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 5494538240 EDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 K>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59053056 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 3382973743616 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 3382973743616 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚332875391008768 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚322 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚41948160 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚385224470839808 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚4293 NGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚75 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 3375926580946 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 125829120 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 4654602240 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚98796899 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚223 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚11741278 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚61478912 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 4517085184 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚381841497096192 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 3383015691776 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚322 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚379587985408000 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚41348068 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚381841514201088 SFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚66776064 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17104896 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚43995273 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚336453053143040 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚20643840 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚46132224 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 H@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚665 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚322 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚46984228 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚382963894949586 _step‚1152 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 122905600 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 IBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚60 _runtime‚3728.226880091 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 831258624 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 5368709120 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚51808378 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚385224529892864 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 3577728910336 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 3577682778112 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 3375967929014 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚385224529892864 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚381841497096192 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654602240 G@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚77 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚43995196 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚11741353 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0€ I‰6Œ2œ!- _wandbruntime‚3728  _runtime‚3728Ó£/%u!ï epoch‚34 % _timestamp‚1.7866623796975126e+09 _runtime‚3728.227337986 0loss/FRt1/assignment_loss‚0.5498367591419182 /loss/FRt2/assignment_loss‚0.485188394587498 trainer/global_step‚13999 _step‚1153 /loss/FRt1/detection_loss‚0.4787296781088881 0loss/FRt2/detection_loss‚0.48768285617922136 &loss/total_loss‚2.0014376880175258€(ëszž!ô &loss/total_loss‚2.0014376880175258 epoch‚34 % _timestamp‚1.7866623796975126e+09 _runtime‚3728.227337986 0loss/FRt1/assignment_loss‚0.5498367591419182 /loss/FRt2/assignment_loss‚0.485188394587498 trainer/global_step‚13999 _step‚1153 /loss/FRt1/detection_loss‚0.4787296781088881 0loss/FRt2/detection_loss‚0.48768285617922136 Îå2Ÿ!- _wandbruntime‚3733  _runtime‚3733¿C¢ß !Ù .Purity/*FRt/FRt_purity‚0.48441341519355774 3Purity/0FRt/event_proportion‚0.3301943838596344 3Purity/1FRt/event_proportion‚0.6297432780265808 'particle/f_score‚0.5452498197555542 0Purity/1FRt/event_purity‚0.47284430265426636 +particle/specificity‚0.6655637621879578 /Purity/2FRt/event_purity‚0.4355721175670624 *jet/accuracy_1_of_1‚0.3148344159126282 2Purity/2FRt/event_proportion‚0.040062315762043 particle/accuracy_1_of_1‚1 )loss/val_total_loss‚2.428970644635779 0Purity/*FRt/event_purity‚0.47070425748825073 2Purity/*FRt/event_proportion‚0.669805645942688 -Purity/2FRt/FRt_purity‚0.5742623805999756 +particle/sensitivity‚0.6026324033737183 *validation_accuracy‚0.2144843488931656 6validation_average_jet_accuracy‚0.3200593888759613 % _timestamp‚1.7866623852270434e+09 Purity/0FRt/event_purity‚1 .Purity/1FRt/FRt_purity‚0.47284430265426636 *jet/accuracy_2_of_2‚0.2144843488931656 (particle/accuracy‚0.6432408094406128 trainer/global_step‚13999 _runtime‚3733.757259282 _step‚1154 *jet/accuracy_1_of_2‚0.5885435938835144 /particle/accuracy_1_of_2‚0.9025124907493591 /particle/accuracy_2_of_2‚0.6508396863937378 epoch‚34+ƒfä¡!Þ _runtime‚3733.757259282 _step‚1154 *jet/accuracy_1_of_2‚0.5885435938835144 /particle/accuracy_1_of_2‚0.9025124907493591 /particle/accuracy_2_of_2‚0.6508396863937378 epoch‚34 .Purity/*FRt/FRt_purity‚0.48441341519355774 3Purity/0FRt/event_proportion‚0.3301943838596344 3Purity/1FRt/event_proportion‚0.6297432780265808 'particle/f_score‚0.5452498197555542 0Purity/1FRt/event_purity‚0.47284430265426636 +particle/specificity‚0.6655637621879578 /Purity/2FRt/event_purity‚0.4355721175670624 *jet/accuracy_1_of_1‚0.3148344159126282 2Purity/2FRt/event_proportion‚0.040062315762043 particle/accuracy_1_of_1‚1 )loss/val_total_loss‚2.428970644635779 0Purity/*FRt/event_purity‚0.47070425748825073 2Purity/*FRt/event_proportion‚0.669805645942688 -Purity/2FRt/FRt_purity‚0.5742623805999756 +particle/sensitivity‚0.6026324033737183 *validation_accuracy‚0.2144843488931656 6validation_average_jet_accuracy‚0.3200593888759613 % _timestamp‚1.7866623852270434e+09 Purity/0FRt/event_purity‚1 .Purity/1FRt/FRt_purity‚0.47284430265426636 *jet/accuracy_2_of_2‚0.2144843488931656 (particle/accuracy‚0.6432408094406128 trainer/global_step‚13999‚ ¯sô”¢!‚(:‰ ò›ùÓ³â­á gpu.0.gpu‚0.0 gpu.0.memory‚0, gpu.0.memoryAllocated‚7.0313262939453125) gpu.0.memoryAllocatedBytes‚ 6039863296 gpu.0.temp‚33.0 gpu.0.powerWatts‚119.703' gpu.0.enforcedPowerLimitWatts‚500.0 gpu.0.powerPercent‚23.9406 gpu.0.smClock‚1410 gpu.0.memoryClock‚1593$ gpu.0.correctedMemoryErrors‚1955# gpu.0.uncorrectedMemoryErrors‚0K¡ÃªŸ£!‚(:” ò›ùÓ«Åÿã# proc.memory.rssMB‚ 2362.08984375* proc.memory.percent‚14.417052268981934 cpu‚27.2236548355771% disk./.usageGB‚487.74455642700195 disk.sda.out‚ 68.4453125" disk.nvme0n1.in‚43914.25390625 disk.dm-1.out‚ 68.4453125# disk.dm-0.out‚59307.70751953125) proc.memory.availableMB‚ 5139.37890625 network.recv‚ 6.983001e+06) disk./.usagePercent‚32.73294642675743 disk.sda3.in‚34886.07421875 disk.sda2.out‚0 disk.drbd2165.in‚0 proc.cpu.threads‚17 network.sent‚ 1.8216489e+07 disk.sda3.out‚ 68.4453125$ disk.nvme0c0n1.in‚43914.25390625 disk.sda.in‚34886.07421875 disk.dm-1.in‚34886.07421875 disk.drbd2165.out‚0 disk.dm-0.in‚43914.25390625$ memory_percent‚68.63172054290771( disk.nvme0c0n1.out‚59307.70751953125 disk.sda1.in‚0 disk.sda1.out‚0 disk.sda2.in‚0& disk.nvme0n1.out‚59307.70751953125±BG2¤!- _wandbruntime‚3746  _runtime‚3746® G3Æ¥!À _step‚1155 % lr-AdamW/pg1‚0.000316094575277856 % lr-AdamW/pg2‚0.000316094575277856 trainer/global_step‚14049 $ _timestamp‚1.786662397786585e+09 _runtime‚3746.316617738ê©Ë¦!Å % lr-AdamW/pg1‚0.000316094575277856 % lr-AdamW/pg2‚0.000316094575277856 trainer/global_step‚14049 $ _timestamp‚1.786662397786585e+09 _runtime‚3746.316617738 _step‚1155ƒ Å5ÞK2§!- _wandbruntime‚3746  _runtime‚37460h(-Á'¨!»O \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚20643840 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚61478912 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 5494538240 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚47172517 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 122306526 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚383522228862976 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚52043081 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚32402577 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚209 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 5494538240 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 3398397164544 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚429 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚47172517 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 845178880 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚262 F?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚60 @7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚5264 M@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚59049984 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚99215598 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚50329600 BDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚334303982583808 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚386920685077504 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚383522245967872 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚32402579 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚322 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 4628521206 % _timestamp‚1.7866623977875767e+09 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚383522245967872 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 5368709120 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚322 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 3398439109632 KBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚4876 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 831258624 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚41348020 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 G>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4876 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚41945088 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 3593865737216 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 4517085184 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚47168230 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 4654602240 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚386920685077504 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚381251988783104 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 5368709120 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 3398439109632 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚52043079 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚384643353922167 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4289 D;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚5264 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚44193342 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚11790687 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 3593815407616 PCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚59049984 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 122905600 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚4287 IBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚78 KDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚60 IDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚386920626027520 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 125829120 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚4289 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 4654602240 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚47168230 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚334303961939968 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚384643295534787 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 4552982528 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚322 IBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚60 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 3398397164544 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚4287 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 4552982528 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 trainer/global_step‚14049 IDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚322 IBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚60 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 3398397164544 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚4287 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 4552982528 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 trainer/global_step‚14049 IDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚334303982583808 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚386920685077504 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚383522245967872 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚32402579 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚322 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 4628521206 % _timestamp‚1.7866623977875767e+09 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚383522245967872 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 5368709120 @8Devic´èe±}eStatsMonitor.on_train_batch_start/segment.all.peak‚322 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 3398439109632 KBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚4876 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 831258624 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚41348020 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 G>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4876 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚41945088 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 3593865737216 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 4517085184 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚47168230 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 4654602240 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚386920685077504 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚381251988783104 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 5368709120 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 3398439109632 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚52043079 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚384643353922167 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4289 D;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚5264 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚44193342 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚11790687 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 3593815407616 PCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚59049984 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 122905600 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚4287 IBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚78 KDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚60 IDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚386920626027520 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 125829120 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚4289 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 4654602240 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚47168230 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚334303961939968 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚384643295534787„ GÁ’52ª!- _wandbruntime‚3746  _runtime‚3746ˆ»gË&«!ÅM NGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚71 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 831258624 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 5368709120 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚322 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚52046634 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 3594109880320 IBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚60 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚41948160 E5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5264 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚383548551200768 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521206 % _timestamp‚1.7866623979504406e+09 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17104896 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚386947238487040 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚44196356 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚262 EDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 DDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚44196283 _step‚1157 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654602240 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚334327029956608 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚381278157537280 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 PDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59053056 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚383548568305664 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 3398670181376 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 5368709120 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚52046632 B9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚5264 G@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚73 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 3594063748096 NADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚58387428 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚381278174576640 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 125829120 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚334327009312768 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17104896 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚99218149 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 4552982528 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚383548568305664 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚99222444 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚52046632 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 3391554363655 NADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚59053056 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚386947238487040 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚337921073060864 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 845178880 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚46132224 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚322 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚4293 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚4295 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚322 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚386947179433984 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 4517085184 _runtime‚3746.480752603 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚99222444 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚262 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 5494538240 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚47171517 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 SFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚66776064 G@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚60 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚4293 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 3398628233216 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚41948160 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2†žÆ½Ð&¬!ÊM JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 3391554363655 NADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚59053056 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚386947238487040 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚337921073060864 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 845178880 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚46132224 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚322 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚4293 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚4295 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚322 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚386947179433984 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 4517085184 _runtime‚3746.480752603 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚99222444 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚262 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 5494538240 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚47171517 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 SFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚66776064 G@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚60 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚4293 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 3398628233216 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚41948160 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 NGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚71 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 831258624 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 5368709120 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚322 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚52046634 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 3594109880320 IBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚60 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚41948160 E5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5264 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚383548551200768 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521206 % _timestamp‚1.7866623979504406e+09 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17104896 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚386947238487040 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚44196356 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚262 EDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 DDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚44196283 _step‚1157 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654602240 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚334327029956608 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚381278157537280 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 PDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59053056 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚383548568305664 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 3398670181376 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 5368709120 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚52046632 B9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚5264 G@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚73 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 3594063748096 NADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚58387428 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚381278174576640 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 125829120 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚334327009312768 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17104896 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚99218149 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 4552982528 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚383548568305664 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚99222444 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚52046632 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1… ¼ø/2­!- _wandbruntime‚3746  _runtime‚3746þ Ft®!î epoch‚35 /loss/FRt1/detection_loss‚0.5265529322677673 /loss/FRt2/detection_loss‚0.5072332003870843 trainer/global_step‚14049 % _timestamp‚1.7866623979515152e+09 _runtime‚3746.481315277 _step‚1158 0loss/FRt1/assignment_loss‚0.5447237406088702 0loss/FRt2/assignment_loss‚0.4496019089449174 %loss/total_loss‚2.028111782208639’ö¬ty¯!ó % _timestamp‚1.7866623979515152e+09 _runtime‚3746.481315277 _step‚1158 0loss/FRt1/assignment_loss‚0.5447237406088702 0loss/FRt2/assignment_loss‚0.4496019089449174 %loss/total_loss‚2.028111782208639 epoch‚35 /loss/FRt1/detection_loss‚0.5265529322677673 /loss/FRt2/detection_loss‚0.5072332003870843 trainer/global_step‚14049† ¬Áa”°!‚(:‰ œùÓžü—á gpu.0.gpu‚7.0 gpu.0.memory‚1, gpu.0.memoryAllocated‚7.0313262939453125) gpu.0.memoryAllocatedBytes‚ 6039863296 gpu.0.temp‚35.0 gpu.0.powerWatts‚136.678' gpu.0.enforcedPowerLimitWatts‚500.0 gpu.0.powerPercent‚27.3356 gpu.0.smClock‚1410 gpu.0.memoryClock‚1593$ gpu.0.correctedMemoryErrors‚1955# gpu.0.uncorrectedMemoryErrors‚0]R!‚(:‘ œùÓ¹Š—ä disk.sda1.in‚0" disk.nvme0n1.in‚43914.25390625& disk.nvme0n1.out‚59312.53662109375 disk.dm-0.in‚43914.25390625) proc.memory.availableMB‚ 5139.86328125# proc.memory.rssMB‚ 2362.08984375 network.recv‚ 7.010189e+06* disk./.usagePercent‚32.721003163270055 disk.drbd2165.in‚0 disk.sda3.in‚34886.07421875 disk.sda3.out‚ 68.703125( disk.nvme0c0n1.out‚59312.53662109375$ memory_percent‚68.62876415252686* proc.memory.percent‚14.417052268981934 network.sent‚ 1.8287937e+07 disk.sda1.out‚0 disk.sda2.in‚0 disk.dm-1.in‚34886.07421875 disk.dm-1.out‚ 68.703125# disk.dm-0.out‚59312.53662109375$ disk.nvme0c0n1.in‚43914.25390625 cpu‚27.22322822407439 disk.sda.out‚ 68.703125 disk.sda2.out‚0 disk.drbd2165.out‚0 proc.cpu.threads‚17# disk./.usageGB‚487.566593170166 disk.sda.in‚34886.07421875›¤Kt2²!- _wandbruntime‚3758  _runtime‚3758Y,*(˳!Å ' lr-AdamW/pg2‚0.00031120591287925786 trainer/global_step‚14099 % _timestamp‚1.7866624100248966e+09 _runtime‚3758.556908654 _step‚1159 ' lr-AdamW/pg1‚0.00031120591287925786T[´Ð´!Ê ' lr-AdamW/pg1‚0.00031120591287925786 ' lr-AdamW/pg2‚0.00031120591287925786 trainer/global_step‚14099 % _timestamp‚1.7866624100248966e+09 _runtime‚3758.556908654 _step‚1159‡ `e!n2µ!- _wandbruntime‚3758  _runtime‚3758ši_¶!»O SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 845178880 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚262 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 122905600 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 BDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4876 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚41945088 JBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚665 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚382561261420544 KDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚60 C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚4289 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 3410025875456 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚429 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚11831524 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 122306526 trainer/global_step‚14099 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚384838362857472 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 122905600 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 831258624 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 3402885601651 F?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚60 _step‚1160 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 3409983930368 D;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚5264 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚32512829 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚99558350 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚209 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚61478912 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚41348020 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 5368709120 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚32512827 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚262 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚47337619 M@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚59049984 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚4287 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚339062431948288 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚322 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 3402926949671 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 5494538240 F>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚384838362857472 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 3409983930368 IDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚322 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 125829120 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚322 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚223 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚47333332 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17104896 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 3410025875456 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚339062509213184 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚20643840 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚52220731 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4289 UHDƒ“Å»eviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚77264896 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚382561278459904 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 5494538240 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 4654602240 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17104896 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 5368709120 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚52220731 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚385964147022195 KBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚4876 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 4628521206 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚47333332 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚44344427 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 3606157989376 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 4517085184 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 125829120 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚322 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚52220729 IBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚76 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚388248388732928 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚388248329682944 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚384838345752576 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 4552982528 @7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚5264 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚56621056 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 3606101368320 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 PCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚58387380 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚4287 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚388248388732928 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚47337619 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 5368709120 _runtime‚3758.557010407 IDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4876 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚41945088 JBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚665 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚382561261420544 KDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚60 C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚4289 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 3410025875456 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚429 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚11831524 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 122306526 trainer/global_step‚14099 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚384838362857472 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 122905600 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 831258624 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 3402885601651 F?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚60 _step‚1160 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 3409983930368 D;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚5264 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚32512829 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚99558350 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚209 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚61478912 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚41348020 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 5368709120 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚32512827 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚262 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚47337619 M@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚59049984 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚4287 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚339062431948288 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚322 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 3402926949671 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 5494538240 F>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚384838362857472 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 3409983930368 IDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚322 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 125829120 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚322 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚223 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚47333332 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17104896 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 3410025875456 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚339062509213184 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚20643840 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚52220731 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4289 UHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚77264896 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚382561278459904 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 5494538240 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 4654602240 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17104896 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 5368709120 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚52220731 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚385964147022195 KBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚4876 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 4628521206 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚47333332 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚44344427 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 3606157989376 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 4517085184 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 125829120 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚322 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚52220729 IBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚76 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚388248388732928 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚388248329682944 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚384838345752576 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 4552982528 @7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚5264 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚56621056 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 3606101368320 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 PCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚58387380 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚4287 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚388248388732928 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚47337619 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 5368709120 _runtime‚3758.557010407 IDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚32515032 >5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5264 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚41948160 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚52224284 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚47340912 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17104896 NGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚63 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚61478912 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 % _timestamp‚1.7866624101875238e+09 EDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521206 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚41348068 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚388274883089408 SFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚70970368 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 _runtime‚3758.717868055 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 3403157522331 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 122306526 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚47336619 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚99565196 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚47340912 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚44347455 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17104896 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚52224284 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚223 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚99560901 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚322 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚322 G@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚65 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 3606395349504 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 3606345022976 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚322 G@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚60 _step‚1161 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚99565196 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚99560901 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚11832421 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 845178880 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 831258624 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚385990563388343 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 122905600 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 122905600 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚388274942142464 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚11832358 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654602240 B9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚5264 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚44347390 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚385990621775771 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚384864685195264 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚384864685195264 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚339085793946112 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚335479398596608 B6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚322 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚388274942142464 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚52224282 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 3410214999040 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚382587447214080 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 4552982528 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚382587464253440 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚41948160 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 5368709120 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 4654602240 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 K>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59053056 D8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚262 D=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚60 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 125829120 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚4293 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 5494538240 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚262 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚4295 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 5368709120 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚262 trainer/global_step‚14099z _"º!ÊM KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654602240 B9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚5264 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚44347390 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚385990621775771 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚384864685195264 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚384864685195264 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚339085793946112 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚335479398596608 B6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚322 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚388274942142464 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚52224282 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 3410214999040 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚382587447214080 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 4552982528 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚382587464253440 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚41948160 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 5368709120 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 4654602240 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 K>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59053056 D8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚262 D=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚60 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 125829120 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚4293 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 5494538240 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚262 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚4295 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 5368709120 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚262 trainer/global_step‚14099 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚52224282 H@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚665 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚429 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 3410256947200 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚339085722975744 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 4517085184 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 5494538240 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚32515032 >5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5264 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚41948160 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚52224284 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚47340912 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17104896 NGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚63 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚61478912 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 % _timestamp‚1.7866624101875238e+09 EDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521206 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚41348068 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚388274883089408 SFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚70970368 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 _runtime‚3758.717868055 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 3403157522331 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 122306526 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚47336619 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚99565196 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚47340912 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚44347455 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17104896 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚52224284 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚223 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚99560901 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚322 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚322 G@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚65 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 3606395349504 ]KDeviceStatsMonRž‹qitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 3606345022976 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚322 G@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚60 _step‚1161 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚99565196 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚99560901 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚11832421 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 845178880 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 831258624 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚385990563388343 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 122905600 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 122905600 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚388274942142464 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚11832358‰ –æôZ2»!- _wandbruntime‚3758  _runtime‚3758«Xr¼!ì 0loss/FRt1/assignment_loss‚0.5337761617722708 1loss/FRt2/assignment_loss‚0.44751194753665124 /loss/FRt2/detection_loss‚0.4902227552091138 trainer/global_step‚14099 # _timestamp‚1.78666241018854e+09 .loss/FRt1/detection_loss‚0.520074397282271 %loss/total_loss‚1.991585261800307 epoch‚35 _runtime‚3758.718348654 _step‚1162Ã9Ï8w½!ñ %loss/total_loss‚1.991585261800307 epoch‚35 _runtime‚3758.718348654 _step‚1162 0loss/FRt1/assignment_loss‚0.5337761617722708 1loss/FRt2/assignment_loss‚0.44751194753665124 /loss/FRt2/detection_loss‚0.4902227552091138 trainer/global_step‚14099 # _timestamp‚1.78666241018854e+09 .loss/FRt1/detection_loss‚0.520074397282271Š …]ÐX–¾!‚(:‹ œùÓ÷à›á gpu.0.gpu‚64.0 gpu.0.memory‚12, gpu.0.memoryAllocated‚7.0313262939453125) gpu.0.memoryAllocatedBytes‚ 6039863296 gpu.0.temp‚37.0 gpu.0.powerWatts‚207.299' gpu.0.enforcedPowerLimitWatts‚500.0 gpu.0.powerPercent‚41.4598 gpu.0.smClock‚1410 gpu.0.memoryClock‚1593$ gpu.0.correctedMemoryErrors‚1955# gpu.0.uncorrectedMemoryErrors‚08’á$”¿!‚(:‰ œùÓë…Üã disk.sda.in‚34886.07421875 disk.sda1.in‚0 disk.sda1.out‚0 disk.sda2.in‚0 disk.nvme0n1.in‚ 43917.625& disk.nvme0n1.out‚59360.21044921875 disk.dm-1.out‚ 68.97265625 proc.cpu.threads‚17 network.recv‚ 7.042482e+06) disk./.usagePercent‚32.71000405556895 disk.drbd2165.in‚0 disk.sda3.out‚ 68.97265625* proc.memory.percent‚14.417052268981934 network.sent‚ 1.8362921e+07$ disk./.usageGB‚487.4026985168457 disk.sda2.out‚0 disk.dm-1.in‚34886.07421875 disk.drbd2165.out‚0 disk.dm-0.in‚ 43917.625# disk.dm-0.out‚59360.21044921875 disk.sda3.in‚34886.07421875 disk.sda.out‚ 68.97265625 disk.nvme0c0n1.in‚ 43917.625( disk.nvme0c0n1.out‚59360.21044921875$ memory_percent‚68.63081455230713) proc.memory.availableMB‚ 5139.52734375# proc.memory.rssMB‚ 2362.08984375 cpu‚27.223011856491322rØ¥‰2À!- _wandbruntime‚3770  _runtime‚3770¸‹°ËÁ!Å ' lr-AdamW/pg1‚0.00030634543340492157 ' lr-AdamW/pg2‚0.00030634543340492157 trainer/global_step‚14149 % _timestamp‚1.7866624221752508e+09 _runtime‚3770.705268854 _step‚1163øðšàÐÂ!Ê _runtime‚3770.705268854 _step‚1163 ' lr-AdamW/pg1‚0.00030634543340492157 ' lr-AdamW/pg2‚0.00030634543340492157 trainer/global_step‚14149 % _timestamp‚1.7866624221752508e+09‹ ¯<Ã2Ã!- _wandbruntime‚3770  _runtime‚3770Š¥†²À'Ä!ºO NFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚223 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚389576033338368 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4289 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚41348020 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 845178880 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚47498434 G>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4876 IBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚67 PIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚65 UHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚62584832 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚336608699219968 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 trainer/global_step‚14149 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚47502721 IBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚60 @7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚5264 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚386154479747072 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚322 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 3618438166016 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 3618396225024 F?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚60 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 4552982528 IDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚32623079 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 5494538240 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚262 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17104896 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 125829120 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚386154462642176 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 3421570696192 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 122905600 KBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚4876 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚429 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚20643840 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚383870551097344 IDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚389576033338368 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 125829120 E8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚99896813 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17104896 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚61478912 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚322 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚383870568136704 JBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚665 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 4517085184 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 3414447412259 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚386154479747072 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚52398381 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 5494538240 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 5368709120 _runtime‚3770.709887258 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚52398381 D;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚5264 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚209 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 3414488760279 $ _timestamp‚1.786662422176182e+09 _step‚1164 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 4654602240 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 3421570696192 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚47498434 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚11872659 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚336608719863808 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚262 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚389576092388352 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚4287 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0ó{«Å'Å!¿O WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17104896 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 125829120 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚386154462642176 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 3421570696192 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 122905600 KBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚4876 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚429 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚20643840 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚383870551097344 IDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚389576033338368 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 125829120 E8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚99896813 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17104896 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚61478912 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚322 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚383870568136704 JBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚665 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 4517085184 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 3414447412259 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚386154479747072 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚52398381 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 5494538240 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 5368709120 _runtime‚3770.709887258 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚52398381 D;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚5264 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚209 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 3414488760279 $ _timestamp‚1.786662422176182e+09 _step‚1164 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 4654602240 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 3421570696192 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚47498434 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚11872659 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚336608719863808 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚262 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚389576092388352 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚4287 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚223 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚389576033338368 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4289 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚41348020 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 845178880 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚47498434 G>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4876 IBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚67 PIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚65 UHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚62584832 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚336608699219968 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 trainer/global_step‚14149 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚47502721 IBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚60 @7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚5264 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚386154479747072 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚322 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 3618438166016 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 3618396225024 F?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚60 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 4552982528 IDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚32623079 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 5494538240 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚262Œ š6—2Æ!- _wandbruntime‚3770  _runtime‚3770;PÎ2!Ç!ÅM ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚20643840 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚387311414875751 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521206 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 5368709120 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚322 G@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚60 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚47501721 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚322 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 3421801764864 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚262 _runtime‚3770.868917247 PDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654602240 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 4552982528 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚389602645797888 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 845178880 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚41348068 % _timestamp‚1.7866624223385825e+09 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚52401934 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚262 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚389602645797888 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚4295 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚52401932 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚52401934 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 3414719332939 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 5494538240 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚99907948 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚389602586744832 I@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚4876 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 3618680285696 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚262 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 4654602240 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 122905600 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚41948160 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 trainer/global_step‚14149 B9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚5264 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 5368709120 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚99907948 H@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚665 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚4293 G@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚56 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚11873522 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 NADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚58387428 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚52401932 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚383896753930240 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚383896736890880 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 122306526 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 125829120 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚47506014 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚99903653 _step‚1165 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚44498750 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚387311473263179 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚47501721 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17104896 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚11873468 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚336631767236608 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚61478912 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚322 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 4517085184 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 125829120 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚32625284 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚209 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 831258624 D=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚60 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 IBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚60 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚4293 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚386180784979968 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚340250447522304 SFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚58387456 BDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59053056 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚386180784979968 NGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚54 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17104896 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚429 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚37743616 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated¼À,{™‚386180802084864 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 122905600 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚44498806 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 5494538240 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 5494538240 >5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5264 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 3421801764864 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 3414677984871 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚322 DDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0:Ð&È!ÊM [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚386180802084864 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 4552982528 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚32625282 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚41948160 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚336631746592768 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 3421843713024 NADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚59053056 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 3421843713024 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚47506014 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 3618642542080 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 125829120 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654602240 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 4552982528 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚389602645797888 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 845178880 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚41348068 % _timestamp‚1.7866624223385825e+09 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚52401934 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚262 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚389602645797888 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚4295 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚52401932 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚52401934 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 3414719332939 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 5494538240 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚99907948 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚389602586744832 I@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚4876 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 3618680285696 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚262 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 4654602240 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 122905600 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚41948160 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 trainer/global_step‚14149 B9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚5264 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 5368709120 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚99907948 H@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚665 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚4293 G@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚56 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚11873522 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 NADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚58387428 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚52401932 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚383896753930240 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚383896736890880 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 122306526 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 125829120 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚47506014 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚99903653 _step‚1165 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚44498750 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚387311473263179 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚47501721 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17104896 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚11873468 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚336631767236608 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚61478912 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚322 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 4517085184 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 125829120 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚32625284 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚209 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 831258624 D=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚60 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 IBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚60 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚4293 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚386180784979968 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚340250447522304 SFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚58387456 BDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59053056 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚386180784979968 NGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚54 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17104896 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚429 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚37743616 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚386180802084864 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 122905600 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚44498806 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 5494538240 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 5494538240 >5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5264 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 3421801764864 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 3414677984871 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚322 DDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚20643840 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚387311414875751 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521206 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 5368709120 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚322 G@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚60 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚47501721 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚322 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 3421801764864 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚262 _runtime‚3770.868917247 PDeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4289 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚52576031 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚4287 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚32733329 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚341391804659712 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 3426009222867 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚32733327 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚223 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 5368709120 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚52576031 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚385179840774144 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚262 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚387470596636672 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 4552982528 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 4552982528 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚44646843 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚390903796043776 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17104896 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚322 @7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚5264 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 122905600 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚61478912 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 5494538240 F?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚60 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚47663536 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚390903796043776 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚337761088503808 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 5494538240 IBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚60 _runtime‚3782.894866756 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚ 100239565 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 4654602240 PCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚58387380 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚262 G>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4876 JDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 4517085184 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 125829120 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚262 D;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚5264 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚341391731589120 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚429 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚388605908384391 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 122306526 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚387470579531776 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 4628521206 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 125829120 F>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚41945088 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚387470596636672 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚ 100243854 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 _step‚1168 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚11913514 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 3630663729152 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17104896 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 3426050570887 $ _timestamp‚1.786662434361657e+09 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚390903736993792 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚322 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 5368709120 KDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚60 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 PIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚71 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 845178880 BDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 4517085184 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 125829120 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚262 D;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚5264 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚341391731589120 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚429 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚388605908384391 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 122306526 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚387470579531776 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 4628521206 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 125829120 F>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚41945088 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚387470596636672 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚ 100243854 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 _step‚1168 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚11913514 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 3630663729152 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17104896 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 3426050570887 $ _timestamp‚1.786662434361657e+09 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚390903736993792 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚322 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 5368709120 KDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚60 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 PIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚71 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 845178880 BDeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4289 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚52576031 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚4287 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚32733329 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚341391804659712 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 3426009222867 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚32733327 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚223 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 5368709120 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚52576031 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚385179840774144 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚262 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚387470596636672 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 4552982528 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 4552982528 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚44646843 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚390903796043776 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17104896 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚322 @7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚5264 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 122905600 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚61478912 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 5494538240 F?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚60 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚47663536 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚390903796043776 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚337761088503808 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 5494538240 IBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚60 _runtime‚3782.894866756 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚ 100239565 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 4654602240 PCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚58387380 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚262 G>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4876 J6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚322 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚47666823 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚387496901869568 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚209 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚ 100250700 B9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚5264 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚11914300 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚262 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 122905600 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 3433388530688 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚ 100246405 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚41937920 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 PDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚32735534 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚32735532 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚322 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654602240 IBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚60 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚ 100246405 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 845178880 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 4654602240 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚41948160 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚47666823 D5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5264 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 4552982528 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚47671116 NGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚64 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 3630955613184 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 125829120 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521206 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚47671116 I@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚4876 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚44649768 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 5368709120 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17104896 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17104896 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚341415028908032 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 G@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚60 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚341415091489792 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚385206043607040 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 125829120 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚387496918974464 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚11914236 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 3426239795479 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 122306526 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 D=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚60 K>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59053056 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚4295 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚387496918974464 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚41948160 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 3630913675264 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 100250700 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚52579582 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 125829120 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚322 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚262 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚387496901869568 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚44649834 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 4517085184 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚52579584 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 NADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚59053056 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚388632266363159 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 5494538240 E5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5264 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 4552982528 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚47671116 NGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚64 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 3630955613184 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 125829120 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521206 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚47671116 I@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚4876 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚44649768 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 5368709120 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17104896 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17104896 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚341415028908032 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 G@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚60 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚341415091489792 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚385206043607040 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 125829120 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚387496918974464 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚11914236 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 3426239795479 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 122306526 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 D=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚60 K>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59053056 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚4295 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚387496918974464 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚41948160 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 3630913675264 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 100250700 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚52579582 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 125829120 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚322 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚262 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚387496901869568 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚44649834 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 4517085184 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚52579584 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 NADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚59053056 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚388632266363159 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 5494538240 E6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚322 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚47666823 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚387496901869568 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚209 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚ 100250700 B9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚5264 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚11914300 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚262 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 122905600 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 3433388530688 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚ 100246405 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚41937920 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 PDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚32735534 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚32735532 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚322 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654602240 IBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚60‘ eJ]ö2×!- _wandbruntime‚3783  _runtime‚3783H®Š(uØ!ï 0loss/FRt2/assignment_loss‚0.4006563245809192 /loss/FRt1/detection_loss‚0.5081059666695992 trainer/global_step‚14199 _runtime‚3783.060765463 0loss/FRt1/assignment_loss‚0.5050970907576391 /loss/FRt2/detection_loss‚0.4603150418064236 &loss/total_loss‚1.8741744238145812 epoch‚35 % _timestamp‚1.7866624345309937e+09 _step‚1170l¥‰ÄzÙ!ô 0loss/FRt1/assignment_loss‚0.5050970907576391 /loss/FRt2/detection_loss‚0.4603150418064236 &loss/total_loss‚1.8741744238145812 epoch‚35 % _timestamp‚1.7866624345309937e+09 _step‚1170 0loss/FRt2/assignment_loss‚0.4006563245809192 /loss/FRt1/detection_loss‚0.5081059666695992 trainer/global_step‚14199 _runtime‚3783.060765463’ ƒID Ú!‚(:• ®œùÓ¨ä‰á gpu.0.gpu‚64.0 gpu.0.memory‚12, gpu.0.memoryAllocated‚7.0313262939453125) gpu.0.memoryAllocatedBytes‚ 6039863296 gpu.0.temp‚36.0 gpu.0.powerWatts‚167.09' gpu.0.enforcedPowerLimitWatts‚500.0) gpu.0.powerPercent‚33.418000000000006 gpu.0.smClock‚1410 gpu.0.memoryClock‚1593$ gpu.0.correctedMemoryErrors‚1955# gpu.0.uncorrectedMemoryErrors‚0$eæTšÛ!‚(: ®œùÓ¾•Ñã) proc.memory.availableMB‚ 5139.47265625 proc.cpu.threads‚17# disk.dm-0.out‚59505.85693359375 network.sent‚ 1.850906e+07 network.recv‚ 7.103291e+06* disk./.usagePercent‚32.692301646591304 disk.sda2.in‚0& disk.nvme0n1.out‚59505.85693359375 disk.dm-1.out‚ 69.484375( disk.nvme0c0n1.out‚59505.85693359375% disk./.usageGB‚487.13891983032227 disk.sda2.out‚0 disk.drbd2165.in‚0 disk.sda3.in‚34886.07421875 disk.sda.in‚34886.07421875 disk.sda.out‚ 69.484375 disk.sda1.in‚0$ memory_percent‚68.63114833831787 disk.dm-1.in‚34886.07421875 disk.dm-0.in‚ 43929.5859375 disk.sda3.out‚ 69.484375# disk.nvme0c0n1.in‚ 43929.5859375# proc.memory.rssMB‚ 2362.08984375* proc.memory.percent‚14.417052268981934 cpu‚27.22237637773973! disk.nvme0n1.in‚ 43929.5859375 disk.drbd2165.out‚0 disk.sda1.out‚00Ú¢ö2Ü!- _wandbruntime‚3795  _runtime‚3795Ñù5ÿÇÝ!Á % lr-AdamW/pg1‚0.000296710270116905 % lr-AdamW/pg2‚0.000296710270116905 trainer/global_step‚14249 % _timestamp‚1.7866624467940032e+09 _runtime‚3795.324006923 _step‚1171ÏýàÌÞ!Æ trainer/global_step‚14249 % _timestamp‚1.7866624467940032e+09 _runtime‚3795.324006923 _step‚1171 % lr-AdamW/pg1‚0.000296710270116905 % lr-AdamW/pg2‚0.000296710270116905“ D­;¼2ß!- _wandbruntime‚3795  _runtime‚3795ÈÄ­Ý à!¿O NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚4287 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚44797302 IBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚76 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 3444744227840 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚32843579 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚47828638 PIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚74 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚61478912 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚262 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚223 PCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚58387380 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚262 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚47828638 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚52753681 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17104896 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚388786696421376 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4289 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 5368709120 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚52753679 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 4552982528 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚388786713526272 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 125829120 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚392231499699200 PCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚59049984 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚386489130450944 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚32843577 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚342556462626816 UHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚68876288 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 3642957250560 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 4517085184 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚322 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚392231440649216 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚392231499699200 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚ 100582317 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚41945088 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 D;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚5264 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚52753681 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚11953649 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚389926759871799 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 122306526 KDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚60 IBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚60 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 122905600 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚47832925 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚342556393750528 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚338913457143808 @7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚5264 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚262 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚4289 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚44797226 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚48232448 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 BDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 3444786172928 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚322 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 4628521206 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚386489147490304 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 F?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚60 JDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4876 M@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚59049984 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚ 100586606 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 3643005483008 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 5494538240 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 5368709120 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚322 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 3444786172928 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 3437571033475 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚209 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 3437612381495 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 4654602240 KBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚4876 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 125829120 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚47832925 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚392231440649216 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 3444744227840 ICDeviceStatsMonitor.on_trai¶œj]èn_batch_start/oversize_segments.allocated‚0 _runtime‚3795.328897801 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 122905600 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚11953723 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚20643840 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 125829120 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚52753679 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 831258624 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚322 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 5494538240 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 5368709120 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚388786696421376 JBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚665 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 845178880 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚4287 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 4552982528 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚41945088 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚338913436499968 _step‚1172 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17104896Ó‹à«Ê'á!ÄO JDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4876 M@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚59049984 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚ 100586606 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 3643005483008 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 5494538240 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 5368709120 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚322 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 3444786172928 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 3437571033475 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚209 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 3437612381495 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 4654602240 KBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚4876 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 125829120 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚47832925 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚392231440649216 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 3444744227840 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 _runtime‚3795.328897801 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 122905600 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚11953723 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚20643840 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 125829120 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚52753679 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 831258624 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚322 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 5494538240 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 5368709120 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚388786696421376 JBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚665 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 845178880 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚4287 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 4552982528 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚41945088 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚338913436499968 _step‚1172 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17104896 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚4287 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚44797302 IBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚76 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 3444744227840 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚32843579 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚47828638 PIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚74 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚61478912 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚262 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚223 PCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚58387380 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚262 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚47828638 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚52753681 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17104896 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚388786696421376 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4289 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 5368709120 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚52753679 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 4552982528 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚388786713526272 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 125829120 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚392231499699200 PCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚59049984 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚386489130450944 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚32843577 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚342556462626816 UHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚68876288 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 3642957250560 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 4517085184 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚322 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚392231440649216 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚392231499699200 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚ 100582317 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚41945088 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 D;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚5264 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚52753681 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚11953649 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚389926759871799 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 122306526 KDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚60 IBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚60 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 122905600 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚47832925 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚342556393750528 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚338913457143808 @7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚5264 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚262 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚4289 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚44797226 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚48232448 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 BDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 3444786172928 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚322 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 4628521206 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚386489147490304 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 F?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚60” ;Æ°2â!- _wandbruntime‚3795  _runtime‚3795Eþ‚Ï&ã!ÉM [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚388813035864064 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 EDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 100593452 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚47836218 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 _step‚1173 PDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 NGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚75 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 845178880 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚322 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚262 % _timestamp‚1.7866624469558196e+09 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 125829120 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚52757234 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚52757234 I@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚4876 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚47831925 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚209 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚61478912 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚262 DDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚4295 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚41948160 EDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚20643840 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 4552982528 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚388813018759168 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 4654602240 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654602240 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚44800353 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 125829120 _runtime‚3795.486096138 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚322 >5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5264 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚32845782 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚388813035864064 B9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚5264 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚342579686481920 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 5368709120 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚388813018759168 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 122905600 NADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚58387428 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 5494538240 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚392257994055680 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17104896 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 4552982528 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚4293 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚386515333283840 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 125829120 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚32845784 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 3444975296512 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 3444975296512 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚46132224 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 IBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚60 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚392258053108736 BDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521206 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚52757232 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚429 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚338936504516608 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚389953176237995 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚262 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚338936483872768 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 H@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚665 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚44800276 NADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚59053056 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚ 100593452 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17104896 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚41348068 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚11954569 trainer/global_step‚14249 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚322 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚386515316244480 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚ 100589157 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚223 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 3643202609152 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚4293 K>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59053056Ÿé \Ô&ä!ÎM E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 3437842954155 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521206 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚52757232 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚429 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚338936504516608 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚389953176237995 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚262 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚338936483872768 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 H@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚665 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚44800276 NADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚59053056 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚ 100593452 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17104896 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚41348068 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚11954569 trainer/global_step‚14249 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚322 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚386515316244480 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚ 100589157 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚223 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 3643202609152 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚4293 K>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59053056 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚388813035864064 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 EDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 100593452 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚47836218 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 _step‚1173 PDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 NGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚75 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 845178880 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚322 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚262 % _timestamp‚1.7866624469558196e+09 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 125829120 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚52757234 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚52757234 I@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚4876 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚47831925 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚209 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚61478912 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚262 DDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚4295 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚41948160 EDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚20643840 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 4552982528 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚388813018759168 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 4654602240 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654602240 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚44800353 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 125829120 _runtime‚3795.486096138 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚322 >5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5264 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚32845782 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚388813035864064 B9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚5264 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚342579686481920 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 5368709120 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚388813018759168 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 122905600 NADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚58387428 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 5494538240 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚392257994055680 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17104896 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 4552982528 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚4293 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚386515333283840 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 125829120 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚32845784 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 3444975296512 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 3444975296512 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚46132224 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 IBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚60 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚392258053108736 BDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚393559144304640 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚41945088 % _timestamp‚1.7866624589685009e+09 JBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚665 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚429 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 F?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚60 trainer/global_step‚14299 G>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4876 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚50329600 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 3456372938752 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚ 100929358 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 122306526 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚391247552971827 C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚4289 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚52931331 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17104896 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 4552982528 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 3456372938752 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚44948042 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 5368709120 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚393559144304640 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚4287 PIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚76 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚340065805139968 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚61478912 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 KDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚60 IBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚60 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 3456330993664 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 3449132844083 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚322 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚390102830415872 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 3655246197248 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 5368709120 PCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚59049984 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚47993740 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 _step‚1176 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 4654602240 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 4654602240 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 4517085184 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 4552982528 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 122905600 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 3655296526848 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚391247611359207 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚322 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚262 JDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚343721051337216 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 5494538240 KBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚4876 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚4287 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 3456330993664 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚20643840 BDeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4289 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚32953829 UHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚70973440 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0Ö/PÊ'í!ÄO F8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚ 100925069 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚47998027 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 PCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚58387380 M@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚59049984 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 122905600 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 831258624 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 @7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚5264 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚390102813310976 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚209 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚343721122310656 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚387798420127744 D;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚5264 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚393559203354624 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚390102830415872 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚44947964 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚32953827 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚393559144304640 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚41945088 % _timestamp‚1.7866624589685009e+09 JBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚665 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚429 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 F?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚60 trainer/global_step‚14299 G>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4876 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚50329600 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 3456372938752 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚ 100929358 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 122306526 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚391247552971827 C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚4289 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚52931331 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17104896 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 4552982528 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 3456372938752 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚44948042 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 5368709120 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚393559144304640 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚4287 PIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚76 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚340065805139968 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚61478912 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 KDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚60 IBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚60 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 3456330993664 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 3449132844083 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚322 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚390102830415872 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 3655246197248 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 5368709120 PCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚59049984 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚47993740 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 _step‚1176 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 4654602240 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 4654602240 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 4517085184 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 4552982528 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 122905600 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 3655296526848 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚391247611359207 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚322 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚262 JDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚343721051337216 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 5494538240 KBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚4876 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚4287 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 3456330993664 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚20643840 BDeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4289 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚32953829 UHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚70973440 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 125829120 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17104896 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚47998027 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360˜ í™>Ù2î!- _wandbruntime‚3807  _runtime‚3807t ÁÏ&ï!ÉM NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚52934884 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚ 100931909 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚429 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚340088873156608 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 122306526 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚322 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚52934884 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚262 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚390129135648768 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚47997027 NGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚72 BDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 % _timestamp‚1.7866624591304817e+09 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚41948160 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 122905600 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 100936204 H@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚665 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚44950946 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 845178880 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚322 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚391274027725403 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 3449404764763 I@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚4876 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚20643840 NADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚58387428 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚322 _runtime‚3807.660805245 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚44951020 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚223 SFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚70970368 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚50326528 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚393585756764160 NADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚59053056 B9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚5264 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 4517085184 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 122905600 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17104896 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚41948160 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 5368709120 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 125829120 DDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521206 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚387824622960640 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 5368709120 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚4295 G@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚74 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 831258624 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚390129152753664 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 3456604010496 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚41348068 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚ 100936204 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 3456562062336 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚391273969337975 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚390129152753664 K>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59053056 E8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 _step‚1177 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 4552982528 EDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17104896 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚262 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚48001320 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 3655541030400 D=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚60 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 3456604010496 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚343744414187008 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚340088852512768 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 G@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚60 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 4654602240 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚11994986 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 IBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚60 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚387824605921280 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 trainer/global_step‚14299 >5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5264 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚48001320 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654602240 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚390129135648768 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚52934882 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚209 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 125829120 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚4293 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚47997027 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚61478912 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 5494538240 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 5494538240 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚11994914 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0WŠOð!ÎM % _timestamp‚1.7866624591304817e+09 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚41948160 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 122905600 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 100936204 H@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚665 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚44950946 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 845178880 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚322 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚391274027725403 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 3449404764763 I@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚4876 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚20643840 NADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚58387428 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚322 _runtime‚3807.660805245 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚44951020 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚223 SFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚70970368 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚50326528 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚393585756764160 NADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚59053056 B9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚5264 TEDeviceSta•ée¡…tsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 4517085184 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 122905600 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17104896 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚41948160 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 5368709120 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 125829120 DDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521206 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚387824622960640 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 5368709120 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚4295 G@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚74 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 831258624 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚390129152753664 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 3456604010496 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚41348068 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚ 100936204 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 3456562062336 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚391273969337975 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚390129152753664 K>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59053056 E8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 _step‚1177 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 4552982528 EDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17104896 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚262 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚48001320 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 3655541030400 D=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚60 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 3456604010496 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚343744414187008 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚340088852512768 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 G@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚60 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 4654602240 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚11994986 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 IBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚60 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚387824605921280 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 trainer/global_step‚14299 >5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5264 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚48001320 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654602240 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚390129135648768 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚52934882 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚209 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 125829120 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚4293 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚47997027 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚61478912 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 5494538240 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 5494538240 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚11994914 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚52934884 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚ 100931909 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚429 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚340088873156608 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 122306526 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚322 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚52934884 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚262 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚390129135648768 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚47997027 NGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚72 BDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0™ ué ú2ñ!- _wandbruntime‚3807  _runtime‚3807i(Ôpò!ê _step‚1178 /loss/FRt1/assignment_loss‚0.573978696048307 $loss/total_loss‚1.98305300233988 epoch‚35 trainer/global_step‚14299 0loss/FRt2/assignment_loss‚0.4351523937170852 /loss/FRt1/detection_loss‚0.4908156848743033 /loss/FRt2/detection_loss‚0.4831062277001845 $ _timestamp‚1.786662459131885e+09 _runtime‚ 3807.661670044©y›uó!ï epoch‚35 trainer/global_step‚14299 0loss/FRt2/assignment_loss‚0.4351523937170852 /loss/FRt1/detection_loss‚0.4908156848743033 /loss/FRt2/detection_loss‚0.4831062277001845 $ _timestamp‚1.786662459131885e+09 _runtime‚ 3807.66167004 _step‚1178 /loss/FRt1/assignment_loss‚0.573978696048307 $loss/total_loss‚1.98305300233988š B¡Œ•ô!‚(:Š ½œùÓ‚ÙŒá gpu.0.gpu‚48.0 gpu.0.memory‚8, gpu.0.memoryAllocated‚7.0313262939453125) gpu.0.memoryAllocatedBytes‚ 6039863296 gpu.0.temp‚36.0 gpu.0.powerWatts‚197.342' gpu.0.enforcedPowerLimitWatts‚500.0 gpu.0.powerPercent‚39.4684 gpu.0.smClock‚1410 gpu.0.memoryClock‚1593$ gpu.0.correctedMemoryErrors‚1955# gpu.0.uncorrectedMemoryErrors‚0Úáƒ÷™õ!‚(:Ž ½œùÓð¸ïâ$ disk.nvme0n1.out‚59509.994140625 disk.dm-0.in‚ 43929.5859375! disk.dm-0.out‚59509.994140625# disk.nvme0c0n1.in‚ 43929.5859375 disk.sda.in‚34886.07421875 disk.sda1.out‚0$ memory_percent‚68.63019466400146 disk.sda2.in‚0 disk.sda2.out‚0 disk.sda.out‚ 69.60546875) proc.memory.availableMB‚ 5139.62890625* proc.memory.percent‚14.417052268981934 cpu‚27.222040286600706 proc.cpu.threads‚17( disk./.usagePercent‚32.7259927501124! disk.nvme0n1.in‚ 43929.5859375 disk.dm-1.in‚34886.07421875 disk.sda3.in‚34886.07421875 disk.sda3.out‚ 69.60546875 disk.sda1.in‚0# proc.memory.rssMB‚ 2362.08984375 network.recv‚ 7.13597e+06% disk./.usageGB‚487.64094161987305 disk.dm-1.out‚ 69.60546875 disk.drbd2165.in‚0 disk.drbd2165.out‚0& disk.nvme0c0n1.out‚59509.994140625 network.sent‚ 1.8584222e+071üç˜2ö!- _wandbruntime‚3819  _runtime‚3819^I±KË÷!Å trainer/global_step‚14349 % _timestamp‚1.7866624713769505e+09 _runtime‚3819.906955483 _step‚1179 ' lr-AdamW/pg1‚0.00028719156076819106 ' lr-AdamW/pg2‚0.00028719156076819106ã7^žÐø!Ê ' lr-AdamW/pg1‚0.00028719156076819106 ' lr-AdamW/pg2‚0.00028719156076819106 trainer/global_step‚14349 % _timestamp‚1.7866624713769505e+09 _runtime‚3819.906955483 _step‚1179› ÔPj#2ù!- _wandbruntime‚3819  _runtime‚3819“…–-Ä'ú!¾O YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚392568462846615 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 3467959704576 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚53108979 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 @7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚5264 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚209 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 5368709120 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 125829120 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 845178880 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚341218173779968 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 122306526 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17104896 JDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4876 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚41945088 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 4654602240 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚20643840 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 5368709120 F?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚60 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚322 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚322 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚48163129 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚4287 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 4654602240 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚391418930200576 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 3467917759488 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 122905600 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 JDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚394886847960064 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17104896 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚392568404459235 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 4628521206 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 125829120 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚53108979 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚4287 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚223 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚341218194423808 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚12034687 C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚4289 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚ 101267821 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚53108981 PIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚69 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 5494538240 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 5368709120 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 3467917759488 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚391418947305472 IBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚60 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚33064079 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 3467959704576 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚391418930200576 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4289 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 122905600 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚394886847960064 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 831258624 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 3460736002711 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚391418947305472 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚394886907010048 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 4552982528 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚41945088 JBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚665 trainer/global_step‚14349 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 KBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚4876 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 3667540070912 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚262 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚394886907010048 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚53108981 UHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚66779136 D;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚5264 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚48163129 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚33064077 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 4517085184 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚41348020 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 5494538240 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚48158842 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚429 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚344885713850880 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 125829120 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚ 101272110 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚344885780630016 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚322 PCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚59049984 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚389107726843904 % _timestamp‚1.7866624713778236e+09 F>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 IBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚71 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚322 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚262 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚48158842 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚12034756 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 3667586206208 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚61478912 PCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚58387380 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚389107709804544 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 5494538240 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚05ûß=É'û!ÃO XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚33064079 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 3467959704576 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚391418930200576 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4289 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 122905600 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚394886847960064 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 831258624 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 3460736002711 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚391418947305472 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚394886907010048 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 4552982528 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚41945088 JBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚665 trainer/global_step‚14349 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 KBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚4876 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 3667540070912 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚262 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚394886907010048 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚53108981 UHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚66779136 D;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚5264 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚48163129 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚33064077 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 4517085184 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚41348020 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 5494538240 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚48158842 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚429 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚344885713850880 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 125829120 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚ 101272110 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚344885780630016 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚322 PCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚59049984 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚389107726843904 % _timestamp‚1.7866624713778236e+09 F>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 IBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚71 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚322 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚262 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚48158842 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚12034756 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 3667586206208 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚61478912 PCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚58387380 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚389107709804544 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 5494538240 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚392568462846615 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 3467959704576 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚53108979 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 @7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚5264 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚209 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 5368709120 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 125829120 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 845178880 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚341218173779968 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 122306526 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17104896 JDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4876 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚41945088 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 4654602240 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚20643840 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 5368709120 F?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚60 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚322 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚322 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚48163129 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚4287 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 4654602240 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚391418930200576 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 3467917759488 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 122905600 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 JDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚394886847960064 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17104896 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚392568404459235 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 4628521206 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 125829120 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚53108979 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚4287 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚223 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚341218194423808 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚12034687 C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚4289 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚ 101267821 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚53108981 PIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚69 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 5494538240 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 5368709120 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 3467917759488 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚391418947305472 IBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚60œ 8M’-2ü!- _wandbruntime‚3820  _runtime‚38207öšóý!ÉM WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚392594879212811 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚394913460419584 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 125829120 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚53112534 NADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚58387428 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚392594820825383 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚262 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 5494538240 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 5368709120 SFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚62581760 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚322 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 BDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 H@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚665 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚33066284 >5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5264 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚48162129 [GDÝ?{Y¹ eviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚391445269643264 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 4552982528 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 831258624 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654602240 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚ 101274661 NGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚66 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17104896 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 5494538240 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 5368709120 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 4654602240 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 5368709120 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚322 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚322 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚53112532 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 125829120 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚322 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚394913460419584 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚391445252538368 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 122905600 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 122905600 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚45101758 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 % _timestamp‚1.7866624715406287e+09 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚41948160 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 3460966575371 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 3468148828160 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 125829120 D=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚60 _step‚1181 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚394913401366528 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚391445252538368 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚4293 G@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚60 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚ 101278956 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 3468190776320 G@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚68 IBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚60 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 3468148828160 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚33066282 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 5494538240 _runtime‚3820.070915134 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚41948160 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚209 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚41348068 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚262 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚4295 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚4293 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚391445269643264 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚223 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 845178880 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚389133912637440 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚429 PDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 101278956 B9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚5264 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚53112532 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 3667830709760 trainer/global_step‚14349 E8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 EDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521206 K>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59053056 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚341241221152768¶åÔ&þ!ÎM UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17104896 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 5494538240 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 5368709120 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 4654602240 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 5368709120 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚322 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚322 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚53112532 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 125829120 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚322 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚394913460419584 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚391445252538368 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 122905600 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 122905600 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚45101758 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 % _timestamp‚1.7866624715406287e+09 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚41948160 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 3460966575371 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 3468148828160 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 125829120 D=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚60 _step‚1181 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚394913401366528 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚391445252538368 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚4293 G@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚60 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚ 101278956 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 3468190776320 G@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚68 IBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚60 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 3468148828160 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚33066282 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 5494538240 _runtime‚3820.070915134 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚41948160 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚209 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚41348068 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚262 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚4295 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚4293 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚391445269643264 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚223 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 845178880 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚389133912637440 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚429 PDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 101278956 B9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚5264 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚53112532 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 3667830709760 trainer/global_step‚14349 E8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 EDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521206 K>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59053056 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚341241221152768 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚392594879212811 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚394913460419584 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 125829120 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚53112534 NADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚58387428 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚392594820825383 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚262 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 5494538240 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 5368709120 SFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚62581760 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚322 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 BDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 H@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚665 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚33066284 >5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5264 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚48162129 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚391445269643264 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 4552982528 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 831258624 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654602240 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚ 101274661 NGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚66 L: g2ÿ!- _wandbruntime‚3820  _runtime‚3820ãp*{v€"ð 0loss/FRt1/assignment_loss‚0.4967583221987056 1loss/FRt2/assignment_loss‚0.44370361862560515 0loss/FRt1/detection_loss‚0.47185398444184645 &loss/total_loss‚1.8509978729254628 /loss/FRt2/detection_loss‚0.4386819476593056 epoch‚35 trainer/global_step‚14349 $ _timestamp‚1.786662471541654e+09 _runtime‚3820.071494509 _step‚1182ÿ >@{"õ 1loss/FRt2/assignment_loss‚0.44370361862560515 0loss/FRt1/detection_loss‚0.47185398444184645 &loss/total_loss‚1.8509978729254628 /loss/FRt2/detection_loss‚0.4386819476593056 epoch‚35 trainer/global_step‚14349 $ _timestamp‚1.786662471541654e+09 _runtime‚3820.071494509 _step‚1182 0loss/FRt1/assignment_loss‚0.4967583221987056ž ðJ©9¡‚"‚(:– ÌœùÓÈØîà gpu.0.gpu‚63.0 gpu.0.memory‚12, gpu.0.memoryAllocated‚7.0313262939453125) gpu.0.memoryAllocatedBytes‚ 6039863296 gpu.0.temp‚36.0 gpu.0.powerWatts‚136.178' gpu.0.enforcedPowerLimitWatts‚500.0) gpu.0.powerPercent‚27.235599999999998 gpu.0.smClock‚1410 gpu.0.memoryClock‚1593$ gpu.0.correctedMemoryErrors‚1955# gpu.0.uncorrectedMemoryErrors‚0¾±³‹¡ƒ"‚(:– ÌœùÓ”¹ãâ$ disk./.usageGB‚487.6389961242676( disk.nvme0c0n1.out‚59847.07470703125 disk.sda.in‚ 34907.9609375" disk.nvme0n1.in‚43941.89453125& disk.nvme0n1.out‚59847.07470703125 disk.dm-1.out‚ 69.95703125* proc.memory.percent‚14.417052268981934 cpu‚27.221697104603596 disk.drbd2165.out‚0# disk.dm-0.out‚59847.07470703125) proc.memory.availableMB‚ 5139.37890625 proc.cpu.threads‚17 network.sent‚ 1.8679817e+07 network.recv‚ 7.168498e+06 disk.drbd2165.in‚0 disk.dm-0.in‚43941.89453125 disk.sda3.in‚ 34907.9609375$ disk.nvme0c0n1.in‚43941.89453125 disk.sda.out‚ 69.95703125 disk.sda1.in‚0* disk./.usagePercent‚32.725862186269936 disk.sda3.out‚ 69.95703125 disk.sda1.out‚0 disk.sda2.in‚0 disk.sda2.out‚0 disk.dm-1.in‚ 34907.9609375$ memory_percent‚68.63172054290771# proc.memory.rssMB‚ 2362.08984375Ì©{2„"- _wandbruntime‚3832  _runtime‚3832+2†Ê…"Ä ' lr-AdamW/pg1‚0.00028247664170584865 ' lr-AdamW/pg2‚0.00028247664170584865 trainer/global_step‚14399 % _timestamp‚1.7866624836062272e+09 _runtime‚ 3832.13626938 _step‚1183¾MKφ"É trainer/global_step‚14399 % _timestamp‚1.7866624836062272e+09 _runtime‚ 3832.13626938 _step‚1183 ' lr-AdamW/pg1‚0.00028247664170584865 ' lr-AdamW/pg2‚0.00028247664170584865Ÿ ¸Þâ_2‡"- _wandbruntime‚3832  _runtime‚3832ÆÍÄ'ˆ"¾O JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚262 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚396214610665472 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚392735047090176 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚48323944 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 PIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚72 @7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚5264 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚4287 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 3479504525312 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚12075858 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 4552982528 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 3679853818368 BDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚396214551615488 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 125829120 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 3472256465299 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 5494538240 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 3479546470400 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚429 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚61478912 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚53286631 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚41945088 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 3479504525312 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚12075786 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚209 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚393889255946643 KDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚60 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚223 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 5494538240 _runtime‚3832.140916105 _step‚1184 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 4552982528 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚53286629 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 F>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚392735047090176 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 845178880 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 4628521206 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 5368709120 IBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚60 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4289 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 5368709120 D;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚5264 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 PCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚58387380 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚390417016520704 PCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚59049984 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚322 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚396214610665472 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 4654602240 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚393889314334023 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚322 C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚4289 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚41945088 JDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4876 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚392735064195072 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚322 JDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚396214551615488 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 125829120 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 3472256465299 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 5494538240 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 3479546470400 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚429 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚61478912 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚53286631 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚41945088 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 3479504525312 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚12075786 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚209 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚393889255946643 KDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚60 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚223 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 5494538240 _runtime‚3832.140916105 _step‚1184 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 4552982528 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚53286629 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 F>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚392735047090176 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 845178880 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 4628521206 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 5368709120 IBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚60 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4289 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 5368709120 D;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚5264 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 PCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚58387380 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚390417016520704 PCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚59049984 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚322 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚396214610665472 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 4654602240 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚393889314334023 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚322 C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚4289 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚41945088 JDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4876 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚392735064195072 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚322 JDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654602240 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚ 101621708 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚ 101617413 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 4552982528 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚392761386532864 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚4293 G@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚57 SFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚56290304 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17104896 H@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚665 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚393915730700219 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 5368709120 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 125829120 % _timestamp‚1.7866624837750008e+09 _step‚1185 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 5368709120 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚392761369427968 NADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚58387428 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 EDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 101621708 G@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚60 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚342393589792768 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 4552982528 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 845178880 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 K>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59053056 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 3479735593984 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚12076618 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 trainer/global_step‚14399 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚48331524 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚45253150 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 P6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚322 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚262 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 3680105140736 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚53290182 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚322 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚41948160 NADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚59053056 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚4295 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚53290184 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521206 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 122905600 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 125829120 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚12076673 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚429 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚342393610436608 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚262 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚396241164075008 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚346073751223808 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 122905600 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚396241164075008 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚33176532 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚41348068 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 125829120 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 5494538240 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚48327231 NGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚55 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 E6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚322 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 4517085184 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚262 D5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5264 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17104896 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚ 101617413 B9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚5264 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚346073694933504 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚35646464 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚322 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚396241105021952 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚48331524 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚48327231 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚392761386532864 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 3472487037911 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 122306526 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 5368709120Ü¥®Ô&Œ"ÎM E6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚322 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 4517085184 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚262 D5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5264 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17104896 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚ 101617413 B9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚5264 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚346073694933504 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚35646464 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚322 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚396241105021952 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚48331524 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚48327231 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚392761386532864 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 3472487037911 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 122306526 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 5368709120 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚390443185274880 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 3479777542144 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚209 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚61478912 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654602240 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚ 101621708 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚ 101617413 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 4552982528 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚392761386532864 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚4293 G@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚57 SFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚56290304 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17104896 H@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚665 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚393915730700219 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 5368709120 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 125829120 % _timestamp‚1.7866624837750008e+09 _step‚1185 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 5368709120 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚392761369427968 NADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚58387428 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 EDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 101621708 G@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚60 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚342393589792768 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 4552982528 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 845178880 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 K>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59053056 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 3479735593984 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚12076618 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 trainer/global_step‚14399 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚48331524 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚45253150 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 P6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚322 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚262 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 3680105140736 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚53290182 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚322 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚41948160 NADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚59053056 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚4295 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚53290184 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521206 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 122905600 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 125829120 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚12076673 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚429 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚342393610436608 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚262 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚396241164075008 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚346073751223808 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 122905600 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚396241164075008 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚33176532 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚41348068 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 125829120 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 5494538240 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚48327231 NGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚55 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0¡ ÁëÄ;2"- _wandbruntime‚3832  _runtime‚38321_ xŽ"ò 0loss/FRt1/assignment_loss‚0.5087196837613537 1loss/FRt2/assignment_loss‚0.42907105403473833 0loss/FRt1/detection_loss‚0.47364646860867365 0loss/FRt2/detection_loss‚0.45839037388185405 epoch‚35 trainer/global_step‚14399 _runtime‚3832.305898187 _step‚1186 &loss/total_loss‚1.8698275802866198 % _timestamp‚1.7866624837760558e+09µR,—}"÷ 0loss/FRt1/assignment_loss‚0.5087196837613537 1loss/FRt2/assignment_loss‚0.42907105403473833 0loss/FRt1/detection_loss‚0.47364646860867365 0loss/FRt2/detection_loss‚0.45839037388185405 epoch‚35 trainer/global_step‚14399 _runtime‚3832.305898187 _step‚1186 &loss/total_loss‚1.8698275802866198 % _timestamp‚1.7866624837760558e+09¢ ®ç׆2"- _wandbruntime‚3837  _runtime‚3837T&1!Ü‘"Ö 2Purity/*FRt/event_proportion‚0.669805645942688 3Purity/0FRt/event_proportion‚0.3301943838596344 *jet/accuracy_1_of_2‚0.6048924326896667 /Purity/*FRt/event_purity‚0.4749101996421814 Purity/0FRt/event_purity‚1 /Purity/1FRt/event_purity‚0.4766243100166321 +Purity/2FRt/FRt_purity‚0.58753502368927 /particle/accuracy_1_of_2‚0.9231123924255371 +particle/sensitivity‚0.6327142715454102 _step‚1187 *jet/accuracy_2_of_2‚0.2214789092540741 /particle/accuracy_2_of_2‚0.6912899017333984 *validation_accuracy‚0.2214789092540741 trainer/global_step‚14399 +jet/accuracy_1_of_1‚0.31576889753341675 particle/accuracy_1_of_1‚1 (particle/accuracy‚0.6401483416557312 'particle/f_score‚0.5551865696907043 7validation_average_jet_accuracy‚0.32163333892822266 epoch‚35 +particle/specificity‚0.6442233920097351 3Purity/1FRt/event_proportion‚0.6297432780265808 -Purity/1FRt/FRt_purity‚0.4766243100166321 2Purity/2FRt/event_proportion‚0.040062315762043 )loss/val_total_loss‚2.422217845803703 /Purity/2FRt/event_purity‚0.4464243948459625 .Purity/*FRt/FRt_purity‚0.48926594853401184 % _timestamp‚1.7866624886763587e+09 _runtime‚3837.206397716q¡Æá’"Û epoch‚35 +particle/specificity‚0.6442233920097351 3Purity/1FRt/event_proportion‚0.6297432780265808 -Purity/1FRt/FRt_purity‚0.4766243100166321 2Purity/2FRt/event_proportion‚0.040062315762043 )loss/val_total_loss‚2.422217845803703 /Purity/2FRt/event_purity‚0.4464243948459625 .Purity/*FRt/FRt_purity‚0.48926594853401184 % _timestamp‚1.7866624886763587e+09 _runtime‚3837.206397716 2Purity/*FRt/event_proportion‚0.669805645942688 3Purity/0FRt/event_proportion‚0.3301943838596344 *jet/accuracy_1_of_2‚0.6048924326896667 /Purity/*FRt/event_purity‚0.4749101996421814 Purity/0FRt/event_purity‚1 /Purity/1FRt/event_purity‚0.4766243100166321 +Purity/2FRt/FRt_purity‚0.58753502368927 /particle/accuracy_1_of_2‚0.9231123924255371 +particle/sensitivity‚0.6327142715454102 _step‚1187 *jet/accuracy_2_of_2‚0.2214789092540741 /particle/accuracy_2_of_2‚0.6912899017333984 *validation_accuracy‚0.2214789092540741 trainer/global_step‚14399 +jet/accuracy_1_of_1‚0.31576889753341675 particle/accuracy_1_of_1‚1 (particle/accuracy‚0.6401483416557312 'particle/f_score‚0.5551865696907043 7validation_average_jet_accuracy‚0.32163333892822266£ ÑEhc”“"‚(:‰ ÛœùÓüå©á gpu.0.gpu‚5.0 gpu.0.memory‚1, gpu.0.memoryAllocated‚7.0313262939453125) gpu.0.memoryAllocatedBytes‚ 6039863296 gpu.0.temp‚35.0 gpu.0.powerWatts‚119.563' gpu.0.enforcedPowerLimitWatts‚500.0 gpu.0.powerPercent‚23.9126 gpu.0.smClock‚1410 gpu.0.memoryClock‚1593$ gpu.0.correctedMemoryErrors‚1955# gpu.0.uncorrectedMemoryErrors‚0WqÍ·V”"‚(:Œ ÛœùÓ­ìŠã disk.sda1.in‚0& disk.nvme0n1.out‚59851.37060546875 disk.sda3.out‚ 70.21484375$ disk.nvme0c0n1.in‚43948.12890625 disk.sda.in‚ 36030.34375$ memory_percent‚68.63055229187012( proc.memory.availableMB‚ 5139.5703125 network.recv‚ 7.199446e+06) disk./.usagePercent‚32.72587447463158 d“1ã‹Aisk.sda2.in‚0 disk.sda2.out‚0 disk.dm-1.in‚ 36030.34375* proc.memory.percent‚14.417052268981934 proc.cpu.threads‚17$ disk./.usageGB‚487.6391792297363" disk.nvme0n1.in‚43948.12890625 disk.dm-1.out‚ 70.21484375( disk.nvme0c0n1.out‚59851.37060546875 disk.sda1.out‚0# proc.memory.rssMB‚ 2362.08984375 cpu‚27.22715407799048 network.sent‚ 1.875333e+07 disk.drbd2165.in‚0 disk.drbd2165.out‚0 disk.dm-0.in‚43948.12890625# disk.dm-0.out‚59851.37060546875 disk.sda3.in‚ 36030.34375 disk.sda.out‚ 70.21484375Íš2•"- _wandbruntime‚3849  _runtime‚3849gä.uÉ–"à & lr-AdamW/pg1‚0.0002777917507951705 & lr-AdamW/pg2‚0.0002777917507951705 trainer/global_step‚14449 % _timestamp‚1.7866625007578502e+09 _runtime‚3849.287869814 _step‚1188ÀjèΗ"È _runtime‚3849.287869814 _step‚1188 & lr-AdamW/pg1‚0.0002777917507951705 & lr-AdamW/pg2‚0.0002777917507951705 trainer/global_step‚14449 % _timestamp‚1.7866625007578502e+09¤ OüÖÿ2˜"- _wandbruntime‚3849  _runtime‚3849ÎÇÇ$Å'™"¿O IBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚77 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 PCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚58387380 BDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚4287 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚394442118299648 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 3495159014912 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚4287 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚322 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 125829120 JDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚397937260209664 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4289 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 3696231436800 KDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚60 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚53524885 G>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4876 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17104896 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚53524885 UHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚68876288 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚61478912 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚41348020 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 5494538240 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 122905600 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚429 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 3487884248008 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚397937319259648 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚ 102036118 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚33325191 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚262 JBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚665 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚395595131285372 _runtime‚3849.291871742 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 3696279669248 C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚4289 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚48511233 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 4552982528 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 831258624 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚48232448 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚397937260209664 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 3487925596028 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚41945088 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚262 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚45451273 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚12126082 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 5494538240 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚41945088 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚343822181367808 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 125829120 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 3495200960000 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 F?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚60 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚48511233 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚223 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚392107188649984xÖKÊ'š"ÄO HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 4654602240 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚394442101194752 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚53524887 PIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚75 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 122306526 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚322 IBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚60 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 3495159014912 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚209 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 5368709120 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚322 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚394442118299648 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚ 102040407 JDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚4287 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚394442118299648 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 3495159014912 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚4287 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚322 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 125829120 JDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚397937260209664 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4289 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 3696231436800 KDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚60 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚53524885 G>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4876 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17104896 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚53524885 UHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚68876288 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚61478912 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚41348020 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 5494538240 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 122905600 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚429 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 3487884248008 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚397937319259648 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚ 102036118 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚33325191 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚262 JBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚665 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚395595131285372 _runtime‚3849.291871742 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 3696279669248 C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚4289 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚48511233 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 4552982528 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 831258624 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚48232448 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚397937260209664 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 3487925596028 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚41945088 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚262 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚45451273 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚12126082 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 5494538240 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚41945088 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚343822181367808 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 125829120 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 3495200960000 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 F?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚60 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚48511233 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚223 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚392107188649984 IBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚77 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 PCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚58387380 B6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚322 D=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚60 PDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚394468440637440 NGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚73 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 B5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5264 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚50326528 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 3696473125376 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 3495390083584 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 3495390083584 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚48514520 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 EDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚45454297 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚12126901 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚4295 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚397963813616128 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654602240 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 831258624 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚262 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚4293 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 4552982528 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 3495432031744 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚20643840 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 3488114820620 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 3696523451904 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚ 102047253 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚53528438 I@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚4876 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 845178880 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚394468423532544 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚12126974 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚343845249384448 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚395621489264140 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚322 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚53528440 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚343845228740608 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 4517085184 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 102047253 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚223 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚395621547651568 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 5494538240 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 5368709120 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚397963872669184 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚ 102042958 H@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚665 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 _runtime‚3849.450148235 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚53528438 DDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521206 $ _timestamp‚1.786662500919847e+09 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚41948160 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 122905600 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 125829120 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 EDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59053056 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚347541772836352 SFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚70970368 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚322 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚262 trainer/global_step‚14449 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 4654602240 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17104896 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚45454372 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚41348068 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 125829120 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚394468440637440 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚397963872669184ç(z¦ï"ÍM JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚262 trainer/global_step‚14449 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 4654602240 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17104896 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚45454372 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚41348068 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 125829120 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚394468440637440 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚397963872669184 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚33327396 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚322 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚322 D=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚60 PDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚394468440637440 NGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚73 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 B5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5264 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚50326528 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 3696473125376 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 3495390083584 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 3495390083584 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚48514520 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 EDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚45454297 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚12126901 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚4295 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚397963813616128 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654602240 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 831258624 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚262 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚4293 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 4552982528 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 3495432031744 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚20643840 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 3488114820620 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 3696523451904 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚ 102047253 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚53528438 I@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚4876 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 845178880 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚394468423532544 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚12126974 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚343845249384448 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚395621489264140 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚322 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚53528440 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚343845228740608 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 4517085184 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 102047253 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚223 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚395621547651568 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 5494538240 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 5368709120 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚397963872669184 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚ 102042958 H@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚665 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 _runtime‚3849.450148235 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚53528438 DDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521206 $ _timestamp‚1.786662500919847e+09 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚41948160 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 122905600 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 125829120 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 EDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59053056 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚347541772836352 SFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚70970368 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚322 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0¦ §äj2ž"- _wandbruntime‚3849  _runtime‚3849Á+ͽwŸ"ñ 0loss/FRt1/assignment_loss‚0.5029561920398152 1loss/FRt2/assignment_loss‚0.41478570390102715 0loss/FRt2/detection_loss‚0.48003202794446803 trainer/global_step‚14449 0loss/FRt1/detection_loss‚0.48639330893056426 &loss/total_loss‚1.8841672328158747 epoch‚36 $ _timestamp‚1.786662500920757e+09 _runtime‚3849.450728913 _step‚1191ð>£q| "ö &loss/total_loss‚1.8841672328158747 epoch‚36 $ _timestamp‚1.786662500920757e+09 _runtime‚3849.450728913 _step‚1191 0loss/FRt1/assignment_loss‚0.5029561920398152 1loss/FRt2/assignment_loss‚0.41478570390102715 0loss/FRt2/detection_loss‚0.48003202794446803 trainer/global_step‚14449 0loss/FRt1/detection_loss‚0.48639330893056426§ eÀ‘ý”¡"‚(:‰ êœùÓ»Á™á gpu.0.gpu‚64.0 gpu.0.memory‚12, gpu.0.memoryAllocated‚7.0313262939453125) gpu.0.memoryAllocatedBytes‚ 6039863296 gpu.0.temp‚36.0 gpu.0.powerWatts‚219.31' gpu.0.enforcedPowerLimitWatts‚500.0 gpu.0.powerPercent‚43.862 gpu.0.smClock‚1410 gpu.0.memoryClock‚1593$ gpu.0.correctedMemoryErrors‚1955# gpu.0.uncorrectedMemoryErrors‚0à7wa™¢"‚(:Ž êœùÓž€ñâ disk.sda3.out‚ 70.73046875( disk.nvme0c0n1.out‚59858.99658203125 disk.sda.in‚ 36030.34375 disk.dm-0.in‚43948.12890625$ memory_percent‚68.63179206848145* proc.memory.percent‚14.417052268981934 proc.cpu.threads‚17 disk.sda2.in‚0 network.sent‚ 1.8825253e+07% disk./.usageGB‚487.63904571533203$ disk.nvme0c0n1.in‚43948.12890625 disk.sda.out‚ 70.73046875 disk.sda1.in‚0& disk.nvme0n1.out‚59858.99658203125 disk.dm-1.in‚ 36030.34375) disk./.usagePercent‚32.72586551436788 disk.sda2.out‚0" disk.nvme0n1.in‚43948.12890625 disk.dm-1.out‚ 70.73046875 disk.drbd2165.in‚0( proc.memory.availableMB‚ 5139.3671875# proc.memory.rssMB‚ 2362.08984375 cpu‚27.22679553797077 network.recv‚ 7.227069e+06 disk.sda3.in‚ 36030.34375 disk.sda1.out‚0 disk.drbd2165.out‚0# disk.dm-0.out‚59858.99658203125œ¹ë#2£"- _wandbruntime‚3861  _runtime‚3861)‘85ɤ"à _runtime‚3861.652864679 _step‚1192 & lr-AdamW/pg1‚0.0002731371889379458 & lr-AdamW/pg2‚0.0002731371889379458 trainer/global_step‚14499 % _timestamp‚1.7866625131209457e+09üȉwÎ¥"È & lr-AdamW/pg1‚0.0002731371889379458 & lr-AdamW/pg2‚0.0002731371889379458 trainer/global_step‚14499 % _timestamp‚1.7866625131209457e+09 _runtime‚3861.652864679 _step‚1192¨ @ü2¦"- _wandbruntime‚3861  _runtime‚3861¿OzûÅ'§"¿O PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚48676335 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚262 G>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4876 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 831258624 PCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚59049984 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 PCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚58387380 _step‚1193 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚399264963865088 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚399264963865088 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 4628521206 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 5494538240 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚322 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚4287 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 4517085184 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚348683112131584 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 845178880 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚48232448 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 3499487406636 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚53702535 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚4287 M@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚59049984 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚393416495366144 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 5368709120 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚48680622 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚209 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚344974550007808 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 122306526 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 5368709120 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 _runtime‚3861.652942496 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚41945088 JDeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4289 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚396915982772780 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚262 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 3506745780736 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 3506787725824 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚33435441 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚223 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚399265022915072 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 122905600 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚45602401 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚262 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚395758235189248 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 4654602240 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 3506787725824 BDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 125829120 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚ 102378870 F>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚393416478326784k ¼áÊ'¨"ÄO TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17104896 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 3506745780736 KBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚4876 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚53702537 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 4552982528 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚45602330 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚12166889 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚344974570651648 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 3708541479936 trainer/global_step‚14499 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚399265022915072 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚41945088 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚348683043255296 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚322 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚322 C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚4289 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚33435443 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 125829120 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 122905600 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚20643840 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚396915924385400 IBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚60 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚395758218084352 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17104896 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚53702537 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚12166958 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 3499446058616 UHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚68876288 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 3708493247488 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 125829120 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚ 102378870 F>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚393416478326784 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚48676335 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚262 G>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4876 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 831258624 PCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚59049984 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 PCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚58387380 _step‚1193 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚399264963865088 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚399264963865088 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 4628521206 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 5494538240 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚322 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚4287 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 4517085184 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚348683112131584 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 845178880 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚48232448 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 3499487406636 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚53702535 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚4287 M@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚59049984 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚393416495366144 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 5368709120 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚48680622 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚209 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚344974550007808 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 122306526 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 5368709120 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 _runtime‚3861.652942496 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚41945088 JDeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4289 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚396915982772780 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚262 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 3506745780736 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 3506787725824 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚33435441 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚223 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚399265022915072 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 122905600 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚45602401 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚262 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚395758235189248 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 4654602240 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 3506787725824 B5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5264 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654602240 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 B9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚5264 IBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚60 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚48683915 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚53706090 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚48679622 G@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚59 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚ 102385710 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚12167742 NADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚58387428 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚4295 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚4293 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 5368709120 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚61478912 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚322 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 D=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚60 _runtime‚3861.814325919 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 122905600 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚45605390 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚429 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚12167685 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 5494538240 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 125829120 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚53706088 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 3506976849408 NADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚59053056 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚395784557527040 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 3708783886336 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚41937920 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚395784557527040 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 3499717979296 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚48679622 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17104896 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 831258624 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚262 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 D8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 125829120 K>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59053056 PDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 102390005 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521206 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚20643840 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚393442664120320 % _timestamp‚1.7866625132839992e+09 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 4552982528 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚4293 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚45605331 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚322 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚262 trainer/global_step‚14499 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 4654602240 H@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚665 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚393442681159680 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 E6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚322 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚262 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 4517085184 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚395784540422144 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 3708741948416 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚396942399138976 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚53706090 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚48683915 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚322 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚209 EDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 5494538240¹ÜÔ&«"ÎM L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 102390005 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521206 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚20643840 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚393442664120320 % _timestamp‚1.7866625132839992e+09 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 4552982528 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚4293 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚45605331 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚322 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚262 trainer/global_step‚14499 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 4654602240 H@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚665 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚393442681159680 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 E6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚322 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚262 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 4517085184 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚395784540422144 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 3708741948416 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚396942399138976 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚53706090 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚48683915 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚322 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚209 EDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 5494538240 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 122905600 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚41348068 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚ 102390005 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 5368709120 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚399291576324608 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚399291576324608 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 3499676631228 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 125829120 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚395784540422144 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚53706088 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚344997597380608 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 122306526 >5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5264 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654602240 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 B9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚5264 IBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚60 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚48683915 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚53706090 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚48679622 G@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚59 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚ 102385710 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚12167742 NADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚58387428 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚4295 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚4293 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 5368709120 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚61478912 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚322 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 D=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚60 _runtime‚3861.814325919 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 122905600 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚45605390 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚429 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚12167685 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 5494538240 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 125829120 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚53706088 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 3506976849408 NADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚59053056 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚395784557527040 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 3708783886336 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚41937920 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚395784557527040 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 3499717979296 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚48679622 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17104896 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 831258624 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚262 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 D8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 125829120 K>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59053056 PDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4876 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 122905600 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚33545693 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚394725785042944 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚400592726570496 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚41348020 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚397074334973952 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 3518332546560 PIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚62 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 5368709120 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 125829120 BDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚400592667520512 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚ 102725911 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚349847737887744 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 M@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚59049984 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚ 102721622 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 3518374491648 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 trainer/global_step‚14549 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚12207280 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 F?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚60 _runtime‚3873.876910175 @7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚5264 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 845178880 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚61478912 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚41945088 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚48841437 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚346126939291648 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 3720798596096 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 3511007869224 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚41945088 PCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚59049984 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚20643840 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 4517085184 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 4654602240 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚397074334973952 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 JDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 JBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚665 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 5494538240 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 3518374491648 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚4287 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 3511049217244 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚322 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚48841437 UHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚62584832 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚53880185 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 3720756655104 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 IBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚60 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚400592667520512 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚53880187 PCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚58387380 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚53880187 IBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚64 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 831258624 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚262 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17104896 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4289 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚398236775872808 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚33545691 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 125829120 C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚4289 D;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚5264 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚48845724 KBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚4876 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚223 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚209 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚349847675302912 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 5494538240 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚400592726570496 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚4287 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚397074352078848 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 4552982528 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚397074352078848 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚346126918647808 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 5368709120 _step‚1197 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 122905600 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚262 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 4654602240 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚48845724WÊþ´«¶"ÃO ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚398236834260188 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 5494538240 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚45752973 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚429 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 4628521206 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚322 KDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚60 G>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4876 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 122905600 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚(,gä$2 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚33545693 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚394725785042944 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚400592726570496 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚41348020 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚397074334973952 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 3518332546560 PIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚62 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 5368709120 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 125829120 BDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚400592667520512 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚ 102725911 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚349847737887744 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 M@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚59049984 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚ 102721622 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 3518374491648 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 trainer/global_step‚14549 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚12207280 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 F?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚60 _runtime‚3873.876910175 @7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚5264 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 845178880 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚61478912 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚41945088 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚48841437 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚346126939291648 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 3720798596096 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 3511007869224 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚41945088 PCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚59049984 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚20643840 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 4517085184 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 4654602240 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚397074334973952 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 JDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 JBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚665 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 5494538240 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 3518374491648 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚4287 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 3511049217244 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚322 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚48841437 UHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚62584832 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚53880185 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 3720756655104 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 IBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚60 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚400592667520512 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚53880187 PCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚58387380 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚53880187 IBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚64 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 831258624 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚262 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17104896 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4289 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚398236775872808 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚33545691 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 125829120 C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚4289 D;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚5264 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚48845724 KBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚4876 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚223 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚209 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚349847675302912 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 5494538240 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚400592726570496 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚4287 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚397074352078848 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 4552982528 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚397074352078848 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚346126918647808 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 5368709120 _step‚1197 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 122905600 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚262 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 4654602240 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚48845724 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 3518332546560 JDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 % _timestamp‚1.7866625255076332e+09 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚48844724 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 PDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521206 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚53883740 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚41948160 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚322 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚398263192238956 trainer/global_step‚14549 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚397100674416640 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 3518605563392 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 3518563615232 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚398263250626384 >5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5264 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚4293 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 3721046048768 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚41948160 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17104896 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚349870961742848 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 3720995722240 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 G@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚60 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚4295 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚61478912 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚262 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚53883738 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654602240 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚209 BDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59053056 E6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚322 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚346149986664448 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚429 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 5494538240 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚ 102732757 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 4552982528 NADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚58387428 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚262 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17104896 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 102732757 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚400619279980032 _runtime‚3874.037975049 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚48849017 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚397100674416640 NGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚67 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚48844724 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚45755870 SFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚70970368­6àÔ&¹"ÎM SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 122306526 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 125829120 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚53883740 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚45755939 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚394751970836480 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 5494538240 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 125829120 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚322 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚346149986664448 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚429 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 5494538240 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚ 102732757 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 4552982528 NADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚58387428 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚262 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17104896 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 102732757 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚400619279980032 _runtime‚3874.037975049 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚48849017 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚397100674416640 NGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚67 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚48844724 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚45755870 SFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚70970368 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚346149966020608 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 % _timestamp‚1.7866625255076332e+09 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚48844724 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 PDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521206 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚53883740 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚41948160 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚322 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚398263192238956 trainer/global_step‚14549 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚397100674416640 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 3518605563392 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 3518563615232 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚398263250626384 >5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5264 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚4293 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 3721046048768 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚41948160 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17104896 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚349870961742848 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 3720995722240 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 G@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚60 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚4295 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚61478912 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚262 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚53883738 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654602240 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚209 BDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59053056 EDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚322 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚45903605 PIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚68 UHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚64681984 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 4654602240 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚429 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚4287 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚223 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 845178880 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚61478912 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 125829120 _runtime‚3886.089875651 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚398390468968448 PCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚59049984 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚401920371175936 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚54057837 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚396035074719744 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚401920430225920 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 4517085184 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 4654602240 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 3529961257472 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 5368709120 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚322 trainer/global_step‚14599 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚54057835 KBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚4876 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 3733086333952 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 5494538240 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚347279307931648 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚262 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚49006539 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚399557627360216 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 5368709120 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 125829120 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚49010826 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚4287 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚401920371175936 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚20643840 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚ 103064374 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚41945088 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚33655943 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚12247732 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚347279287287808 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚322 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 5494538240 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 5368709120 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚398390451863552 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 3733042295808 BDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4876 F>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17104896 JDeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4289 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 4628521206 _step‚1201 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 3529961257472 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 122905600 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 3522611027852 F?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚60 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 4552982528 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚396035057680384 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 3522569679832 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17104896yÚ8CÊ'Ä"ÄO D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚322 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚45903605 PIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚68 UHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚64681984 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 4654602240 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚429 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚4287 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚223 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 845178880 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚61478912 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 125829120 _runtime‚3886.089875651 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚398390468968448 PCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚59049984 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚401920371175936 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚54057837 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚396035074719744 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚401920430225920 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 4517085184 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 4654602240 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 3529961257472 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 5368709120 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚322 trainer/global_step‚14599 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚54057835 KBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚4876 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 3733086333952 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 5494538240 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚347279307931648 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚262 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚49006539 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚399557627360216 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 5368709120 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 125829120 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚49010826 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚4287 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚401920371175936 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚20643840 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚ 103064374 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚41945088 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚33655943 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚12247732 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚347279287287808 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚322 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 5494538240 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 5368709120 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚398390451863552 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 3733042295808 BDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4876 F>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17104896 JDeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4289 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 4628521206 _step‚1201 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 3529961257472 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 122905600 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 3522611027852 F?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚60 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 4552982528 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚396035057680384 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 3522569679832 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17104896 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 122905600 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚398390468968448 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚209 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚4289 @7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚5264 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚49006539 JBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚665 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 5494538240 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚12247664 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚398390451863552 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 3529919312384 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚351012329583616 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 122306526 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚41945088 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 831258624 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚44038144 KDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚60 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360± ö€×a2Å"- _wandbruntime‚3886  _runtime‚38863L¿§Ï&Æ"ÉM K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚45906618 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚41948160 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚12248540 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 5494538240 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 3530192329216 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚209 % _timestamp‚1.7866625377229557e+09 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚4293 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 3733282411520 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 5494538240 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 125829120 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654602240 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚396061260513280 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚ 103071214 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 122905600 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 3733332738048 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 D=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚60 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚398416791306240 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 4517085184 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚54061390 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚351035617072128 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 845178880 H@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚665 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚396061243473920 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚262 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 3530150381056 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 103075509 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚20643840 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚322 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚ 103071214 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚33658146 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 122306526 trainer/global_step‚14599 K>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59053056 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚54061390 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚401946983635456 I@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚4876 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚50326528 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚223 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚54061388 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚398416774201344 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 3522841600512 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17104896 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚347302334660608 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚322 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 3522800252444 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 5368709120 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚429 EDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521206 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚322 P8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚ 103075509 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚401946983635456 NGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚68 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚399584043726364 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 125829120 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 4552982528 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚262 _step‚1202 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚49014119 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚4293 NADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚59053056 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚54061388 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚398416791306240 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚41348068 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚49014119 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚49009826 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 5368709120 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 125829120 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 5494538240 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚322 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 _runtime‚3886.253282437 SFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚70970368 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 831258624 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 5368709120 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚4295 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 3530192329216 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚45906688 D5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5264 B9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚5264 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚12248472 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 G@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚60 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚401946924582400 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 4552982528 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚399584102113792 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚262x•›¹ Ç"ÎM REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚41948160 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚12248540 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 5494538240 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 3530192329216 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚209 % _timestamp‚1.7866625377229557e+09 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚4293 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 3733282411520 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 5494538240 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 125829120 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654602240 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚396061260513280 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚ 103071214 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 122905600 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 3733332738048 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 D=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚60 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚398416791306240 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 4517085184 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚54061390 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚351035617072128 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 845178880 H@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚665 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚396061243473920 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚262 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 3530150381056 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 103075509 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚20643840 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚322 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚ 103071214 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚33658146 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 122306526 trainer/global_step‚14599 K>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59053056 NADeviceStatsMonitor.on_train_batch_end/actŸp{Aive.large_pool.allocated‚54061390 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚401946983635456 I@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚4876 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚50326528 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚223 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚54061388 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚398416774201344 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 3522841600512 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17104896 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚347302334660608 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚322 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 3522800252444 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 5368709120 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚429 EDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521206 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚322 P8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚ 103075509 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚401946983635456 NGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚68 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚399584043726364 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 125829120 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 4552982528 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚262 _step‚1202 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚49014119 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚4293 NADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚59053056 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚54061388 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚398416791306240 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚41348068 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚49014119 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚49009826 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 5368709120 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 125829120 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 5494538240 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚322 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 _runtime‚3886.253282437 SFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚70970368 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 831258624 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 5368709120 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚4295 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 3530192329216 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚45906688 D5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5264 B9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚5264 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚12248472 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 G@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚60 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚401946924582400 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 4552982528 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚399584102113792 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚262 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚45906618² tt›2È"- _wandbruntime‚3886  _runtime‚3886ê§ZuÉ"ï &loss/total_loss‚1.9621094060410602 epoch‚36 $ _timestamp‚1.786662537724022e+09 0loss/FRt2/assignment_loss‚0.4538084580519306 0loss/FRt1/detection_loss‚0.47731675985905775 trainer/global_step‚14599 _runtime‚3886.253854067 _step‚1203 0loss/FRt1/assignment_loss‚0.5495243343435543 /loss/FRt2/detection_loss‚0.4814598537865177;VzÊ"ô /loss/FRt2/detection_loss‚0.4814598537865177 &loss/total_loss‚1.9621094060410602 epoch‚36 $ _timestamp‚1.786662537724022e+09 0loss/FRt2/assignment_loss‚0.4538084580519306 0loss/FRt1/detection_loss‚0.47731675985905775 trainer/global_step‚14599 _runtime‚3886.253854067 _step‚1203 0loss/FRt1/assignment_loss‚0.5495243343435543³ ¿×ƒ-2Ë"- _wandbruntime‚3898  _runtime‚3898Ð6Ò ÊÌ"Ä ' lr-AdamW/pg1‚0.00025935845737012225 ' lr-AdamW/pg2‚0.00025935845737012225 trainer/global_step‚14649 $ _timestamp‚1.786662549817027e+09 _runtime‚3898.348878637 _step‚1204©ô ÏÍ"É trainer/global_step‚14649 $ _timestamp‚1.786662549817027e+09 _runtime‚3898.348878637 _step‚1204 ' lr-AdamW/pg1‚0.00025935845737012225 ' lr-AdamW/pg2‚0.00025935845737012225´ #N(ò2Î"- _wandbruntime‚3898  _runtime‚3898%ÔoµÄ'Ï"¾O PCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚59049984 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4289 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚33766191 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚348431655927808 BDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚403248074831360 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚262 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 3541548023296 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚41945088 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 3534131490440 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 5368709120 _runtime‚3898.348954731 G>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4876 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚399706568753152 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚33766193 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚322 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚49171641 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚46054688 @7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚5264 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚49175928 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 122905600 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 5368709120 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚209 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 3745355022336 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚403248074831360 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 4552982528 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 3541506078208 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 _step‚1205 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17104896 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚54235485 PIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚72 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚352177031593984 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚54235485 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚403248133881344 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 831258624 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚49175928 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚262 JDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚403248133881344 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚322 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 5494538240 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 125829120 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚4287 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚61478912 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 4628521206 IBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚74 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 3541506078208 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚50329600 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚4287 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚352176960620544 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 125829120ñÿ?.É'Ð"ÃO ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 3534172838460 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚399706585858048 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚399706568753152 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚322 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚4289 D;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚5264 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚429 F>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚403248133881344 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚322 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 5494538240 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 125829120 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚4287 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚61478912 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 4628521206 IBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚74 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 3541506078208 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚50329600 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚4287 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚352176960620544 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 125829120 PCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚59049984 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4289 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚33766191 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚348431655927808 BDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚403248074831360 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚262 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 3541548023296 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚41945088 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 3534131490440 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 5368709120 _runtime‚3898.348954731 G>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4876 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚399706568753152 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚33766193 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚322 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚49171641 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚46054688 @7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚5264 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚49175928 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 122905600 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 5368709120 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚209 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 3745355022336 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚403248074831360 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 4552982528 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 3541506078208 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 _step‚1205 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17104896 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚54235485 PIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚72 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚352177031593984 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚54235485 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚403248133881344 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 831258624 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚49175928 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚262 JÑ2Ñ"- _wandbruntime‚3898  _runtime‚3898Ù«¹´Ò"ÉM EDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚4295 NADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚59053056 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚403274628237824 SFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚77261824 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521206 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 3534403411120 _step‚1206 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚ 1011âë3413966 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚49174928 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚348454703300608 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 125829120 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17104896 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚352200324756480 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 B9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚5264 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 831258624 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚322 BDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚46057701 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚12289305 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 3745600812032 G@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚83 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 845178880 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 5494538240 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚262 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 trainer/global_step‚14649 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚399732891090944 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚49174928 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚46057784 NGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚81 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 4517085184 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚399732891090944 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚348454723944448 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚61478912 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 5368709120 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚322 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 125829120 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 125829120 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 103418261 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚4293 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚400904895213772 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 5368709120 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654602240 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 3534362063052 IBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚60 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 3745544194048 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚322 _runtime‚3898.507925251 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚12289386 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚400904953601200 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚397370533150720 G@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚60 >5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5264 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚41948160 EDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59053056 PDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 B9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚5264 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 831258624 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚322 BDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚46057701 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚12289305 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 3745600812032 G@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚83 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 845178880 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 5494538240 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚262 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 trainer/global_step‚14649 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚399732891090944 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚49174928 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚46057784 NGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚81 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 4517085184 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚399732891090944 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚348454723944448 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚61478912 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 5368709120 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚322 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 125829120 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 125829120 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 103418261 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚4293 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚400904895213772 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 5368709120 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654602240 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 3534362063052 IBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚60 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 3745544194048 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚322 _runtime‚3898.507925251 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚12289386 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚400904953601200 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚397370533150720 G@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚60 >5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5264 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚41948160 EDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59053056 PDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚4295 NADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚59053056 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚403274628237824 SFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚77261824 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521206 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 3534403411120¶ '§±2Ô"- _wandbruntime‚3898  _runtime‚3898ªoÈvÕ"ð 0loss/FRt1/detection_loss‚0.46194742595988364 %loss/total_loss‚2.004465202487654 % _timestamp‚1.7866625499786484e+09 _runtime‚3898.508455724 0loss/FRt1/assignment_loss‚0.5669881773902832 0loss/FRt2/detection_loss‚0.48835429334253044 epoch‚36 trainer/global_step‚14649 _step‚1207 0loss/FRt2/assignment_loss‚0.4871753057949565Â>Vˆ{Ö"õ 0loss/FRt2/assignment_loss‚0.4871753057949565 0loss/FRt1/detection_loss‚0.46194742595988364 %loss/total_loss‚2.004465202487654 % _timestamp‚1.7866625499786484e+09 _runtime‚3898.508455724 0loss/FRt1/assignment_loss‚0.5669881773902832 0loss/FRt2/detection_loss‚0.48835429334253044 epoch‚36 trainer/global_step‚14649 _step‚1207· f6ÿÍ”×"‚(:‰ —ùÓÃ÷“á gpu.0.gpu‚0.0 gpu.0.memory‚0, gpu.0.memoryAllocated‚7.0313262939453125) gpu.0.memoryAllocatedBytes‚ 6039863296 gpu.0.temp‚34.0 gpu.0.powerWatts‚119.563' gpu.0.enforcedPowerLimitWatts‚500.0 gpu.0.powerPercent‚23.9126 gpu.0.smClock‚1410 gpu.0.memoryClock‚1593$ gpu.0.correctedMemoryErrors‚1955# gpu.0.uncorrectedMemoryErrors‚0¢ið‹Ø"‚(:€ —ùÓ¤¯éâ disk.sda1.in‚0 disk.sda1.out‚0 proc.cpu.threads‚17) disk./.usagePercent‚32.72586909847336% disk./.usageGB‚487.63909912109375 disk.sda3.in‚ 36030.34375 disk.sda3.out‚ 71.41796875 disk.sda2.in‚0 disk.nvme0n1.in‚ 43951.984375$ disk.nvme0n1.out‚59875.783203125 disk.drbd2165.in‚0& disk.nvme0c0n1.out‚59875.783203125 disk.dm-0.in‚ 43951.984375# memory_percent‚68.6380386352539& proc.memory.availableMB‚ 5138.34375# proc.memory.rssMB‚ 2362.08984375 cpu‚27.225664538226603 disk.sda.in‚ 36030.34375 disk.sda2.out‚0 disk.dm-1.in‚ 36030.34375 disk.dm-1.out‚ 71.41796875 disk.drbd2165.out‚0! disk.dm-0.out‚59875.783203125* proc.memory.percent‚14.417052268981934 network.sent‚ 1.9044041e+07 network.recv‚ 7.317369e+06" disk.nvme0c0n1.in‚ 43951.984375 disk.sda.out‚ 71.41796875¬€<2Ù"- _wandbruntime‚3910  _runtime‚3910S\ŸÊÚ"Ä ' lr-AdamW/pg1‚0.00025482818149777056 ' lr-AdamW/pg2‚0.00025482818149777056 trainer/global_step‚14699 $ _timestamp‚1.786662562125851e+09 _runtime‚3910.657900701 _step‚1208¥wÏÛ"É _runtime‚3910.657900701 _step‚1208 ' lr-AdamW/pg1‚0.00025482818149777056 ' lr-AdamW/pg2‚0.00025482818149777056 trainer/global_step‚14699 $ _timestamp‚1.786662562125851e+09¸ 0—Ç2Ü"- _wandbruntime‚3910  _runtime‚3910AÔ)þÅ'Ý"¿O PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚49341030 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17104896 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 3553092844032 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚49341030 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 3757577069568 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 4628521206 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚262 F?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚60 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚46205700 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 122905600 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚349584024567808 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚61478912 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚322 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 845178880 JBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚665 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 122306526 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 4552982528 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 3545734649068 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚401022702747648 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 125829120 M@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚59049984 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚349584045211648 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚54413137 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚404575837536768 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚404575778486784 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚429 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚353341668416512 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 3545693301048 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 5494538240 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚ 103749878 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17104896 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 5368709120 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚401022685642752 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 122905600 PIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚72 UHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚66779136 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 5494538240 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 4517085184 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 3553092844032 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚41945088 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚54413137 JDeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4289 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚12329331 BDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚398653637033984 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 4654602240 JDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚401022702747648 KBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚4876 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚223 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚46135296 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚322 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 5368709120 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚ 103754167 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚322 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚49336743 G>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4876 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚4287 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 831258624 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 125829120 KDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚60 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚4287 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚404575778486784 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚353341601637376 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚41348020 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚262 IBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚60 % _timestamp‚1.7866625621265497e+09 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚54413135 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 4552982528 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚12329259 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 PCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚59049984 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚398653654073344 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 5494538240 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0‹ù¬æ²Þ"ÄO N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 5494538240 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚49341030 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17104896 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 3553092844032 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚49341030 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 3757577069568 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 4628521206 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚262 F?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚60 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚46205700 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 122905600 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚349584024567808 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚61478912 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚322 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 845178880 JBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚665 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 122306526 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 4552982528 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 3545734649068 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚401022702747648 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 125829120 M@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚59049984 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚349584045211648 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚54413137 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚404575837536768 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚404575778486784 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚429 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚353341668416512 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 3545693301048 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 5494538240 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚ 103749878 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17104896 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 5368709120 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚401022685642752 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 122905600 PIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚72 UHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚66779136 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 5494538240 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 4517085184 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 3553092844032 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚41945088 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚54413137 JDeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4289 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚12329331 BDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚398653637033984 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 4654602240 JDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚401022702747648 KBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚4876 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚223 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚46135296 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚322 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 5368709120 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚ 103754167 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚322 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚49336743 G>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4876 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚4287 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 831258624 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 125829120 KDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚60 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚4287 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚404575778486784 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚353341601637376 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚41348020 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚262 IBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚60 % _timestamp‚1.7866625621265497e+09 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚54413135 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 4552982528 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚12329259 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 PCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚59049984 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚398653654073344¹ Dn2ß"- _wandbruntime‚3910  _runtime‚3910£ðÍÏ&à"ÉM C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚322 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 3553323912704 NGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚67 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚46208810 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚4295 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17104896 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 103761013 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚262 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚ 103756718 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚49344323 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚353364957999104 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 3553365860864 E8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 E6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚322 H@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚665 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚401049025085440 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚41948160 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 4552982528 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚20643840 NADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚58387428 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 5494538240 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚401049007980544 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚33878646 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 5494538240 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 G@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚69 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 3757821379584 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚262 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17104896 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚402225805088608 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚322 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654602240 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚401049025085440 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚ 103756718 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚402225746701180 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚54416688 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚223 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚404602390946304 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 3553365860864 trainer/global_step‚14699 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 3553323912704 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚ 103761013 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚49344323 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚12330162 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚262 G@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚60 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚209 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 5494538240 K>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59053056 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 5368709120 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚46208741 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 122905600 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚49340030 B9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚5264 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚353364893320192 BDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521206 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 125829120 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚33878648 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 3545923873660 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚4293 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 3545965221728 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 % _timestamp‚1.7866625622879236e+09 >5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5264 D6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚322 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 4517085184M›ÎÔ&á"ÎM E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚429 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 E6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚322 H@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚665 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚401049025085440 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚41948160 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 4552982528 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚20643840 NADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚58387428 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 5494538240 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚401049007980544 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚33878646 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 5494538240 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 G@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚69 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 3757821379584 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚262 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17104896 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚402225805088608 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚322 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654602240 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚401049025085440 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚ 103756718 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚402225746701180 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚54416688 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚223 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚404602390946304 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 3553365860864 trainer/global_step‚14699 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 3553323912704 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚ 103761013 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚49344323 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚12330162 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚262 G@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚60 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚209 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 5494538240 K>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59053056 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 5368709120 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚46208741 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 122905600 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚49340030 B9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚5264 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚353364893320192 BDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521206 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 125829120 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚33878648 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 3545923873660 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚4293 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 3545965221728 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 % _timestamp‚1.7866625622879236e+09 >5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5264 D6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚322 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 4517085184 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚322 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 3553323912704 NGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚67 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚46208810 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚4295 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17104896 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 103761013 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚262 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚ 103756718 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚49344323 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚353364957999104 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 3553365860864 EDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4876 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 5368709120 F?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚60 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚405903541192192 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚41945088 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚49506132 M@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚59049984 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚49501845 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚403520181822440 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚399962943750144 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚405903541192192 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚402338819637248 D;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚5264 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚48232448 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 5494538240 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 125829120 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 3564721554944 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 122905600 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚262 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚429 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚322 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚223 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚12370218 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚12370152 BDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4289 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚322 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚403520240209820 % _timestamp‚1.7866625743746827e+09 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚354506226711552 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚350736413851648 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 4517085184 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚46356843 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 _step‚1213 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚33986693 PCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚59049984 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚405903482142208 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 4552982528 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚20643840 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 4628521206 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚262 IBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚60 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚405903482142208 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 4654602240 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 3564721554944 JDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚41945088 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 3557296459676 C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚4289 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚402338802532352 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚33986691 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 845178880 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17104896 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚209 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17104896 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 122905600 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 5494538240 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚4287 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚ 104096919 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚54590787 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚54590785 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0‹·©:Ê'ì"ÄO F>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4289 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚322 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚403520240209820 % _timestamp‚1.7866625743746827e+09 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚354506226711552 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚350736413851648 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 4517085184 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚46356843 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 _step‚1213 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚33986693 PCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚59049984 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚405903482142208 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 4552982528 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚20643840 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 4628521206 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚262 IBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚60 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚405903482142208 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 4654602240 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 3564721554944 JDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚41945088 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 3557296459676 C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚4289 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚402338802532352 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚33986691 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 845178880 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17104896 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚209 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17104896 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 122905600 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 5494538240 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚4287 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚ 104096919 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚54590787 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚54590785 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 3557255111656 trainer/global_step‚14749 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚41348020 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 5368709120 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 3564679609856 G>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4876 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 5368709120 F?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚60 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚405903541192192 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚41945088 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚49506132 M@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚59049984 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚49501845 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚403520181822440 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚399962943750144 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚405903541192192 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚402338819637248 D;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚5264 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚48232448 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 5494538240 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 125829120 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 3564721554944 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 122905600 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚262 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚429 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚322 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚223 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚12370218 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚12370152 B6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚322 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚49509425 G@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚71 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 NADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚58387428 _runtime‚3923.071514355 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 3564952626688 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚33988896 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚50326528 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 DDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59053056 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 4552982528 G@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚60 _step‚1214 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654602240 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 3564910678528 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚399989112504320 EDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 NADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚59053056 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚322 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚403546598188588 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 3557527032336 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 5368709120 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 125829120 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚322 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚41948160 trainer/global_step‚14749 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 3564910678528 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 5494538240 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚405930094601728 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚402365124870144 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 3557485684268 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 122905600 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 122905600 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17104896 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚54594338 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚12370981 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 831258624 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚4293 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚405930035548672 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚223 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 3770125518848 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 PDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 5368709120 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚49505132 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚49505132 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚46359808 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚ 104099470 B9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚5264 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 5494538240 IBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚60 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17104896 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚402365124870144 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚41948160 I@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚4876 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚354529586743296 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 4517085184 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚54594340 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 H@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚665 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚49509425 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚33988898 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 125829120 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚262 % _timestamp‚1.7866625745410514e+09 NGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚69 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 3770075192320 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 122306526 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚322 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 4654602240 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 845178880 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚61478912 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚41348068 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 5494538240 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 104103765 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521206 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚399989129543680 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚262 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚ 104103765 >5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5264 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1ì>_ ¸ï"ÎM T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚405930094601728 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚322 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚49509425 G@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚71 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 NADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚58387428 _runtime‚3923.071514355 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 3564952626688 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚33988896 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚50326528 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 DDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59053056 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 4552982528 G@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚60 _step‚1214 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654602240 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 3564910678528 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚399989112504320 EDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 NADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚59053056 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚322 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚403546598188588 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 3557527032336 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 5368709120 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 125829120 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚322 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚41948160 trainer/global_step‚14749 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 3564910678528 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 5494538240 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚405930094601728 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚402365124870144 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 3557485684268 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 122905600 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 122905600 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17104896 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚54594338 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚12370981 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 831258624 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚4293 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚405930035548672 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚223 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 3770125518848 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 PDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 5368709120 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚49505132 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚49505132 K>DeviceStatsMonitor.on_train_batch_end/inactive_splité%H .all.freed‚46359808 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚ 104099470 B9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚5264 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 5494538240 IBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚60 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17104896 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚402365124870144 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚41948160 I@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚4876 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚354529586743296 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 4517085184 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚54594340 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 H@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚665 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚49509425 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚33988898 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 125829120 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚262 % _timestamp‚1.7866625745410514e+09 NGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚69 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 3770075192320 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 122306526 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚322 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 4654602240 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 845178880 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚61478912 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚41348068 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 5494538240 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 104103765 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521206 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚399989129543680 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚262 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚ 104103765 >5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5264 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 BDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚407231185797632 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚322 JBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚665 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚355670871620096 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 5368709120 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚54768437 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚49666947 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚61478912 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚403654919421952 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚34096943 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚355670934204928 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 5368709120 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 125829120 PCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚59049984 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17104896 _runtime‚3935.210367504 JDeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4289 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚351888761847808 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚404841091697228 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 5368709120 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚403654936526848 IBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚61 BDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4876 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 122905600 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 4654602240 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 4552982528 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚351888782491648 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 3576308320768 PIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚59 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚322 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚401272216387584 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 4517085184 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 125829120 F>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 3576266375680 PCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚58387380 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚322 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚262 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚34096941 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚403654919421952 D;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚5264 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚46507550 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚20643840 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 KDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚60 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚54768437 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 3782109772288 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 4628521206 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 F?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚60 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚223 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 $ _timestamp‚1.786662586680104e+09 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 3576308320768 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚429 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 3782151713280 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚262 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚403654936526848 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 4552982528 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚12410609 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚407231185797632 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 3576266375680 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚41348020 C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚4289 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17104896 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚49671234 UHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚62584832 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚41940992 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 845178880 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚401272233426944 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚4287 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚407231244847616 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 5494538240…4å&É'ú"ÃO \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚403654919421952 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚34096943 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚355670934204928 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 5368709120 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 125829120 PCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚59049984 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17104896 _runtime‚3935.210367504 JDeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4289 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚351888761847808 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚404841091697228 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 5368709120 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚403654936526848 IBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚61 BDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4876 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 122905600 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 4654602240 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 4552982528 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚351888782491648 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 3576308320768 PIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚59 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚322 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚401272216387584 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 4517085184 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 125829120 F>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 3576266375680 PCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚58387380 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚322 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚262 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚34096941 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚403654919421952 D;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚5264 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚46507550 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚20643840 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 KDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚60 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚54768437 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 3782109772288 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 4628521206 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 F?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚60 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚223 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 $ _timestamp‚1.786662586680104e+09 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 3576308320768 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚429 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 3782151713280 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚262 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚403654936526848 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 4552982528 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚12410609 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚407231185797632 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 3576266375680 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚41348020 C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚4289 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17104896 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚49671234 UHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚62584832 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚41940992 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 845178880 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚401272233426944 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚4287 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚407231244847616 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 5494538240 _step‚1217 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚41945088 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚ 104439671 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 122306526 IBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚60 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚407231244847616 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚ 104435382 @7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚5264 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 122905600 KBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚4876 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 trainer/global_step‚14799 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 3568816922264 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚262 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚41945088 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚322 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 4654602240 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚4287 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚209 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 831258624 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 5494538240 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 5494538240 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 3568858270284 JDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚407231185797632 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚322 JBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚665 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚355670871620096 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 5368709120 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚54768437 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚49666947 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚61478912Á !áº2û"- _wandbruntime‚3935  _runtime‚3935Ïê©yü"ÉM trainer/global_step‚14799 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521206 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚401298419220480 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 DDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚46510545 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚ 104446517 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 4552982528 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654602240 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚49670234 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚209 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 3569047494876 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚4295 I@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚4876 G@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚60 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 3576497444352 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚ 104442222 NGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚74 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚44035072 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚262 >5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5264 G@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚76 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚12411473 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚41348068 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 5368709120 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚ 104442222 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚4293 PDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59053056 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚404867449675996 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 5494538240 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚403681241759744 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 3782352509440 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚404867508063424 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚403681258864640 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚54771988 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚355694161730048 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 104446517 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚34099146 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚322 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚262 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚401298402181120 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 3569088842944 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 4654602240 B9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚5264 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚20643840 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 831258624 _runtime‚3935.374396996 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17104896 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚429 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚322 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocat¹17Ved‚ 3576539392512 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 125829120 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚54771988 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 122905600 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚12411399 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 E6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚322 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚49670234 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚61478912 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 122306526 _step‚1218 EDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚223 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚407257798257152 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚41948160 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚41948160 SFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚64678912 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 845178880 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 4517085184 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 5494538240 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚34099148 B5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5264 G@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚76 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚12411473 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚41348068 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 5368709120 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚ 104442222 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚4293 PDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59053056 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚404867449675996 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 5494538240 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚403681241759744 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 3782352509440 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚404867508063424 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚403681258864640 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚54771988 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚355694161730048 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 104446517 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚34099146 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚322 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚262 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚401298402181120 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 3569088842944 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 4654602240 B9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚5264 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚20643840 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 831258624 _runtime‚3935.374396996 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17104896 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚429 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚322 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 3576539392512 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 125829120 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚54771988 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 122905600 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚12411399 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 E6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚322 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚49670234 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚61478912 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 122306526 _step‚1218 EDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚223 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚407257798257152 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚41948160 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚41948160 SFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚64678912 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 845178880 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 4517085184 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 5494538240 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚34099148 BDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521206 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚401298419220480 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 DDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚46510545 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚ 104446517 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 4552982528 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654602240 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚49670234 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚209 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 3569047494876 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚4295 I@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚4876 G@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚60 ƒ¸Je2þ"- _wandbruntime‚3935  _runtime‚3935(àå±wÿ"ñ 1loss/FRt1/assignment_loss‚0.48910304997744714 0loss/FRt2/assignment_loss‚0.3837173058359553 0loss/FRt1/detection_loss‚0.47825010860382355 /loss/FRt2/detection_loss‚0.4894061114241087 &loss/total_loss‚1.8404765758413348 % _timestamp‚1.7866625868450603e+09 _step‚1219 epoch‚36 trainer/global_step‚14799 _runtime‚3935.374851845Ôfù|€#ö 1loss/FRt1/assignment_loss‚0.48910304997744714 0loss/FRt2/assignment_loss‚0.3837173058359553 0loss/FRt1/detection_loss‚0.47825010860382355 /loss/FRt2/detection_loss‚0.4894061114241087 &loss/total_loss‚1.8404765758413348 % _timestamp‚1.7866625868450603e+09 _step‚1219 epoch‚36 trainer/global_step‚14799 _runtime‚3935.374851845à DÕíã2#- _wandbruntime‚3940  _runtime‚39400²­Ü‚#Ö /Purity/*FRt/event_purity‚0.4709751605987549 _runtime‚3940.370785929 _step‚1220 trainer/global_step‚14799 2Purity/*FRt/event_proportion‚0.669805645942688 .Purity/*FRt/FRt_purity‚0.48489952087402344 0Purity/1FRt/event_purity‚0.47305646538734436 .Purity/1FRt/FRt_purity‚0.47305646538734436 *jet/accuracy_1_of_1‚0.3121359646320343 /particle/accuracy_2_of_2‚0.6901869773864746 'particle/accuracy‚0.639346182346344 +particle/sensitivity‚0.6219218373298645 2Purity/2FRt/event_proportion‚0.040062315762043 epoch‚36 /Purity/2FRt/event_purity‚0.4366370737552643 particle/accuracy_1_of_1‚1 *validation_accuracy‚0.2176525741815567 )loss/val_total_loss‚2.455261206031415 3Purity/1FRt/event_proportion‚0.6297432780265808 -Purity/2FRt/FRt_purity‚0.5768031477928162 *jet/accuracy_2_of_2‚0.2176525741815567 'particle/f_score‚0.5503638386726379 6validation_average_jet_accuracy‚0.3175775408744812 % _timestamp‚1.7866625918407388e+09 3Purity/0FRt/event_proportion‚0.3301943838596344 *jet/accuracy_1_of_2‚0.5876733064651489 .particle/accuracy_1_of_2‚0.918239951133728 +particle/specificity‚0.6489178538322449 Purity/0FRt/event_purity‚1S‚{,áƒ#Û +particle/sensitivity‚0.6219218373298645 2Purity/2FRt/event_proportion‚0.040062315762043 epoch‚36 /Purity/2FRt/event_purity‚0.4366370737552643 particle/accuracy_1_of_1‚1 *validation_accuracy‚0.2176525741815567 )loss/val_total_loss‚2.455261206031415 3Purity/1FRt/event_proportion‚0.6297432780265808 -Purity/2FRt/FRt_purity‚0.5768031477928162 *jet/accuracy_2_of_2‚0.2176525741815567 'particle/f_score‚0.5503638386726379 6validation_average_jet_accuracy‚0.3175775408744812 % _timestamp‚1.7866625918407388e+09 3Purity/0FRt/event_proportion‚0.3301943838596344 *jet/accuracy_1_of_2‚0.5876733064651489 .particle/accuracy_1_of_2‚0.918239951133728 +particle/specificity‚0.6489178538322449 Purity/0FRt/event_purity‚1 /Purity/*FRt/event_purity‚0.4709751605987549 _runtime‚3940.370785929 _step‚1220 trainer/global_step‚14799 2Purity/*FRt/event_proportion‚0.669805645942688 .Purity/*FRt/FRt_purity‚0.48489952087402344 0Purity/1FRt/event_purity‚0.47305646538734436 .Purity/1FRt/FRt_purity‚0.47305646538734436 *jet/accuracy_1_of_1‚0.3121359646320343 /particle/accuracy_2_of_2‚0.6901869773864746 'particle/accuracy‚0.639346182346344Ä ?Œß¡„#‚(:– ÄùÓ«Øöà gpu.0.gpu‚64.0 gpu.0.memory‚12, gpu.0.memoryAllocated‚7.0313262939453125) gpu.0.memoryAllocatedBytes‚ 6039863296 gpu.0.temp‚35.0 gpu.0.powerWatts‚177.639' gpu.0.enforcedPowerLimitWatts‚500.0) gpu.0.powerPercent‚35.527800000000006 gpu.0.smClock‚1410 gpu.0.memoryClock‚1593$ gpu.0.correctedMemoryErrors‚1955# gpu.0.uncorrectedMemoryErrors‚0]¶E›…#‚(:„ ÄùÓ¬÷¾â disk.sda1.out‚0 cpu‚27.23021343303695 network.recv‚ 7.415265e+06$ disk./.usageGB‚488.2641181945801 disk.dm-1.in‚ 36075.8671875 disk.drbd2165.out‚0 disk.dm-0.out‚60667.52734375 disk.sda1.in‚0) disk./.usagePercent‚32.76781465292096 disk.sda2.in‚0 disk.sda2.out‚0 disk.dm-0.in‚ 43959.8984375# disk.nvme0c0n1.in‚ 43959.8984375* proc.memory.percent‚14.036083221435547 proc.cpu.threads‚17 network.sent‚ 1.928972e+07 disk.dm-1.out‚ 72.13671875 disk.drbd2165.in‚0 disk.sda3.out‚ 72.13671875 disk.sda.out‚ 72.13671875$ memory_percent‚68.25613975524902( proc.memory.availableMB‚ 5200.9140625! proc.memory.rssMB‚ 2299.671875! disk.nvme0n1.in‚ 43959.8984375# disk.nvme0n1.out‚60667.52734375 disk.sda3.in‚ 36075.8671875% disk.nvme0c0n1.out‚60667.52734375 disk.sda.in‚ 36075.8671875À±†[2†#- _wandbruntime‚3952  _runtime‚3952gÐ]šÊ‡#Ä ' lr-AdamW/pg1‚0.00024142933129681917 ' lr-AdamW/pg2‚0.00024142933129681917 trainer/global_step‚14849 $ _timestamp‚1.786662603895053e+09 _runtime‚3952.425091045 _step‚12212êHψ#É ' lr-AdamW/pg1‚0.00024142933129681917 ' lr-AdamW/pg2‚0.00024142933129681917 trainer/global_step‚14849 $ _timestamp‚1.786662603895053e+09 _runtime‚3952.425091045 _step‚1221Å % à2‰#- _wandbruntime‚3952  _runtime‚3952”‘¼Å'Š#¿O N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 5494538240 JDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚41348020 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 5368709120 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 M@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚59049984 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚55006693 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚223 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 831258624 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 125829120 trainer/global_step‚14849 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚322 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚262 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚49858523 KBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚4876 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚406546850261197 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚12461648 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚20643840 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 3584486052993 _step‚1222 C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚4289 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 4517085184 % _timestamp‚1.7866626038960001e+09 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 125829120 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚4287 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚ 104865216 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚429 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚405361990631424 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚41945088 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 PCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚58387380 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚405361973526528 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚34247805 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚357138902003200 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 3591920865280 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚408953953441792 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 3798501207552 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚61478912 BDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4289 UHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚68876288 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 5494538240 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 @7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚5264 D;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚5264 _runtime‚3952.429931782 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 4552982528 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 125829120 JBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚665 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚49858523 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚405361990631424 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚46709379 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚406546908648577 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚402962422595584 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚402962405556224 JDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4876 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚408953894391808 PCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚59049984_$`‹#ÄO WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚55006691 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚353340421439488 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚322 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 122905600 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚4287 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 4628521206 IBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚60 IBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚76 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 4552982528 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 3591962810368 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 3591962810368 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚48232448 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 PIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚74 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 5494538240 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚322 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17104896 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚41348020 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 5368709120 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 M@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚59049984 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚55006693 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚223 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 831258624 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 125829120 trainer/global_step‚14849 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚322 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚262 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚49858523 KBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚4876 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚406546850261197 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚12461648 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚20643840 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 3584486052993 _step‚1222 C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚4289 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 4517085184 % _timestamp‚1.7866626038960001e+09 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 125829120 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚4287 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.al Ž\Fjlocated‚ 104865216 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚429 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚405361990631424 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚41945088 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 PCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚58387380 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚405361973526528 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚34247805 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚357138902003200 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 3591920865280 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚408953953441792 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 3798501207552 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚61478912 BDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4289 UHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚68876288 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 5494538240 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 @7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚5264 D;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚5264 _runtime‚3952.429931782 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 4552982528 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 125829120 JBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚665 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚49858523 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚405361990631424 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚46709379 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚406546908648577 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚402962422595584 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚402962405556224 JDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4876 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚408953894391808 PCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚59049984 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 5494538240 JDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚46712344 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚405388295864320 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 3592193882112 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚357162265704960 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 3584716625653 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 122306526 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 125829120 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚55010246 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚408980447798272 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚405388312969216 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚4293 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 D8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 3592193882112 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚12462334 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚402988608389120 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 5494538240 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚262 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 5368709120 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 3592151933952 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 5494538240 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚322 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚322 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚49857523 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚405388312969216 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚ 104867767 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚353363448168448 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚61478912 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚402988591349760 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 125829120 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚55010246 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚429 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚406573325014773 G@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚60 D=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚60 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚49857523 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚357162186345984 NADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚58387428 K>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59053056 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521206 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17104896 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚12462416 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 EDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 trainer/global_step‚14849 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654602240 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚55010244 E6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚322 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 4552982528 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 3798738177536 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 122905600 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚58715136 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚406573266627345 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚262 NGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚82 B5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5264 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17104896 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚353363468812288 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚322 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚41948160 I@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚4876 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 5368709120 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚ 104872062 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚49861816 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 104872062 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 3584675277585 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚4295 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 4552982528 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚405388295864320 H@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚665 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚46712428 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚41348068 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 3592151933952 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚223 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 5368709120 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚262 _runtime‚3952.589134341 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 4654602240 NADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚59053056 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 5494538240¤Æœ~Ó&Ž#ÍM trainer/global_step‚14849 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654602240 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚55010244 E6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚322 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 4552982528 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 3798738177536 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 122905600 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚58715136 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚406573266627345 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚262 NGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚82 B5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5264 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17104896 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚353363468812288 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚322 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚41948160 I@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚4876 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 5368709120 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚ 104872062 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚49861816 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 104872062 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 3584675277585 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚4295 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 4552982528 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚405388295864320 H@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚665 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚46712428 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚41348068 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 3592151933952 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚223 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 5368709120 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚262 _runtime‚3952.589134341 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 4654602240 NADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚59053056 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 5494538240 PDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚46712344 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚405388295864320 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 3592193882112 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚357162265704960 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 3584716625653 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 122306526 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 125829120 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚55010246 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚408980447798272 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚405388312969216 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚4293 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 D8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 3592193882112 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚12462334 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚402988608389120 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 5494538240 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚262 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 5368709120 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 3592151933952 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 5494538240 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚322 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚322 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚49857523 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚405388312969216 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚ 104867767 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚353363448168448 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚61478912 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚402988591349760 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 125829120 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚55010246 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚429 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚406573325014773 G@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚60 D=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚60 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚49857523 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚357162186345984 NADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚58387428 K>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59053056 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521206 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17104896 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚12462416 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 EDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0Ç Íó9u2#- _wandbruntime‚3952  _runtime‚3952íù^'u#ï 0loss/FRt1/detection_loss‚0.48863077420127465 epoch‚37 trainer/global_step‚14849 _step‚1224 /loss/FRt2/detection_loss‚0.4600388704356945 &loss/total_loss‚1.8615849635681523 % _timestamp‚1.7866626040598629e+09 _runtime‚3952.589659103 0loss/FRt1/assignment_loss‚0.4899223619417942 /loss/FRt2/assignment_loss‚0.422992956989389ë z‘#ô 0loss/FRt1/assignment_loss‚0.4899223619417942 /loss/FRt2/assignment_loss‚0.422992956989389 0loss/FRt1/detection_loss‚0.48863077420127465 epoch‚37 trainer/global_step‚14849 _step‚1224 /loss/FRt2/detection_loss‚0.4600388704356945 &loss/total_loss‚1.8615849635681523 % _timestamp‚1.7866626040598629e+09 _runtime‚3952.589659103È |¢]À ’#‚(:• ÓùÓ’ößá gpu.0.gpu‚28.0 gpu.0.memory‚6, gpu.0.memoryAllocated‚7.0313262939453125) gpu.0.memoryAllocatedBytes‚ 6039863296 gpu.0.temp‚33.0 gpu.0.powerWatts‚118.997' gpu.0.enforcedPowerLimitWatts‚500.0) gpu.0.powerPercent‚23.799400000000002 gpu.0.smClock‚1410 gpu.0.memoryClock‚1593$ gpu.0.correctedMemoryErrors‚1955# gpu.0.uncorrectedMemoryErrors‚0‚qø/œ“#‚(:‘ ÓùÓ°–Àã disk.sda3.in‚ 37233.546875$ disk.nvme0c0n1.in‚44007.44140625" disk.nvme0n1.in‚44007.44140625 disk.drbd2165.in‚0% disk./.usageGB‚488.26411056518555( disk.nvme0c0n1.out‚60672.62158203125 disk.sda.out‚ 72.67578125 disk.sda1.in‚0 disk.drbd2165.out‚0# disk.dm-0.out‚60672.62158203125$ memory_percent‚68.25602054595947 proc.cpu.threads‚17 network.sent‚ 1.9352579e+07 network.recv‚ 7.433576e+06) disk./.usagePercent‚32.76781414090589 disk.sda.in‚ 37233.546875 disk.sda1.out‚0 disk.sda2.in‚0) proc.memory.availableMB‚ 5200.93359375! proc.memory.rssMB‚ 2299.671875 disk.sda3.out‚ 72.67578125 disk.sda2.out‚0& disk.nvme0n1.out‚60672.62158203125 disk.dm-1.in‚ 37233.546875 disk.dm-1.out‚ 72.67578125 disk.dm-0.in‚44007.44140625* proc.memory.percent‚14.036083221435547 cpu‚27.229897998529747Ü‹;2”#- _wandbruntime‚3964  _runtime‚3964´ŸQïË•#Å trainer/global_step‚14899 % _timestamp‚1.7866626162965317e+09 _runtime‚3964.826532438 _step‚1225 ' lr-AdamW/pg1‚0.00023702799657404012 ' lr-AdamW/pg2‚0.00023702799657404012Û_’Ж#Ê ' lr-AdamW/pg1‚0.00023702799657404012 ' lr-AdamW/pg2‚0.00023702799657404012 trainer/global_step‚14899 % _timestamp‚1.7866626162965317e+09 _runtime‚3964.826532438 _step‚1225É n«q2—#- _wandbruntime‚3964  _runtime‚3964)b„¬˜#¾O L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 4654602240 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 4552982528 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 3603549576192 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 831258624 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 3596006515581 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 5494538240 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚410281657097216 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚223 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚20643840 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 3810817591808 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚404271695233024 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚354492790079488 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 5368709120 @7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚5264 PCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚59049984 JDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 125829120 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚429 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚358303534600704 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 3596047863601 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 4654602240 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4289 JBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚665 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚12502477 F>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚34358057 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚262 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 122306526 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 125829120 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚406678107521024 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚322 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚50019338 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17104896 PCDeviceStatsMonitor.on_train_batchzo.“¿_start/allocation.small_pool.freed‚50019338 C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚4289 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 3603549576192 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚41945088 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 122905600 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚354492769435648 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 5494538240 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 125829120 % _timestamp‚1.7866626162974257e+09 JDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4876 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 5368709120 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 122905600 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚12502552 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚322 F?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚60 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚55184343 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚46860532 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 845178880 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚52426752 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 4517085184 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚55184343 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 KDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚60 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 3810765165056 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚41348020 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 5368709120 IBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚60 _step‚1226 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚410281598047232 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17104896 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 5494538240 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 _runtime‚ 3964.83088869 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚50023625 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚4287 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚262 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚ 105207968 KBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚4876 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚406678090416128 D;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚5264 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚34358055 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚46860609 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚ 105203679 M@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚59049984 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚55184341 PIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚75 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚209 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚407867760135985 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚322 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚406678090416128 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 3603507631104 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚410281657097216 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 4552982528 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚41945088 UHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚73070592 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚404271712272384 trainer/global_step‚14899 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚406678107521024 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 3603507631104 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚358303607671296 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 BDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚410281598047232 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17104896 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 5494538240 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 _runtime‚ 3964.83088869 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚50023625 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚4287 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚262 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚ 105207968 KBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚4876 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚406678090416128 D;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚5264 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚34358055 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚46860609 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚ 105203679 M@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚59049984 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚55184341 PIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚75 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚209 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚407867760135985 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚322 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚406678090416128 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 3603507631104 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚410281657097216 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 4552982528 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚41945088 UHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚73070592 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚404271712272384 trainer/global_step‚14899 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚406678107521024 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 3603507631104 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚358303607671296 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 BDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 125829120 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚429 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚358303534600704 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 3596047863601 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 4654602240 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4289 JBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚665 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚12502477 F>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚34358057 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚262 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 122306526 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 125829120 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚406678107521024 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚322 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚50019338 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17104896 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚50019338 C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚4289 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 3603549576192 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚41945088 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 122905600 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚354492769435648 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 5494538240 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 125829120 % _timestamp‚1.7866626162974257e+09 JDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4876 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 5368709120 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 122905600 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚12502552 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚322 F?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚60 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚55184343 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚46860532 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 845178880Ê ì_^2š#- _wandbruntime‚3964  _runtime‚3964X¸¡ýÏ&›#ÉM ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚406704412753920 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 5494538240 IBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚60 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚322 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚ 105210519 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 3603780647936 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚34360260 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚12503303 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 B5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5264 P6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚322 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚4295 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚410308151453696 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚50026918 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 122306526 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚322 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17104896 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚209 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 trainer/global_step‚14899 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚4293 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 3811020681728 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 _step‚1227 K>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59053056 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 4552982528 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚50026918 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 122905600 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654602240 H@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚665 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 125829120 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚4293 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚35646464 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚262 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 4552982528 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚ 105210519 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚55187896 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚46863623 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚223 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚406704429858816 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17104896 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚358326893780480 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 831258624 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521206 _runtime‚3964.989127716 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 4654602240 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚61478912 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚407894176502181 % _timestamp‚1.7866626164588244e+09 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 3596278436261 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚410308210506752 EDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚262 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚34360262 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚41348068 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚55187896 I@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚4876 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚407894118114753 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚404297898065920 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 845178880 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 3811056328192 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 4517085184 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 3596237088193 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚50022625 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚404297881026560 D=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚60 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚ 105214814 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚12503361 SFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚56290304 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚55187894 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 3603738699776 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚20643840 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 5368709120 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚406704429858816 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 105214814 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚322 EDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚46863563 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0œÇbžTœ#ÎM ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚406704412753920 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 5494538240 IBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚60 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚322 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚ 105210519 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 3603780647936 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚34360260 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚12503303 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 B5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5264 P6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚322 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚4295 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚410308151453696 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚50026918 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 122306526 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚322 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17104896 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚209 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 trainer/global_step‚14899 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚4293 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 3811020681728 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 _step‚1227 K>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59053056 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 4552982528 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚50026918 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 122905600 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654602240 H@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚665 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 125829120 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚4293 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚35646464 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚262 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 4552982528 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚ 105210519 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚55187896 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚46863623 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚223 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚406704429858816 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17104896 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚358326893780480 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 831258624 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521206 _runtime‚3964.989127716 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 4654602240 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚61478912 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚407894176502181 % _timestamp‚1.7866626164588244e+09 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 3596278436261 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.fU;“€reed‚0 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚410308210506752 EDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚262 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚34360262 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚41348068 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚55187896 I@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚4876 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚407894118114753 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚404297898065920 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 845178880 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 3811056328192 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 4517085184 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 3596237088193 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚50022625 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚404297881026560 D=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚60 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚ 105214814 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚12503361 SFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚56290304 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚55187894 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 3603738699776 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚20643840 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 5368709120 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚406704429858816 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 105214814 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚322 EDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚46863563 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0Ë ž42#- _wandbruntime‚3964  _runtime‚3964Ë;ºDvž#ð 0loss/FRt1/assignment_loss‚0.5131681613004684 0loss/FRt2/assignment_loss‚0.4054877167005202 /loss/FRt1/detection_loss‚0.5122695684535421 &loss/total_loss‚1.8918227941542773 epoch‚37 trainer/global_step‚14899 _runtime‚3964.989665252 _step‚1228 0loss/FRt2/detection_loss‚0.46089734769974666 % _timestamp‚1.7866626164598448e+09±’ýØ{Ÿ#õ trainer/global_step‚14899 _runtime‚3964.989665252 _step‚1228 0loss/FRt2/detection_loss‚0.46089734769974666 % _timestamp‚1.7866626164598448e+09 0loss/FRt1/assignment_loss‚0.5131681613004684 0loss/FRt2/assignment_loss‚0.4054877167005202 /loss/FRt1/detection_loss‚0.5122695684535421 &loss/total_loss‚1.8918227941542773 epoch‚37Ì õ[Tâ– #‚(:‹ âùÓåûá gpu.0.gpu‚61.0 gpu.0.memory‚11, gpu.0.memoryAllocated‚7.0313262939453125) gpu.0.memoryAllocatedBytes‚ 6039863296 gpu.0.temp‚35.0 gpu.0.powerWatts‚203.792' gpu.0.enforcedPowerLimitWatts‚500.0 gpu.0.powerPercent‚40.7584 gpu.0.smClock‚1410 gpu.0.memoryClock‚1593$ gpu.0.correctedMemoryErrors‚1955# gpu.0.uncorrectedMemoryErrors‚0¬,½Ð™¡#‚(:Ž âùÓì…äâ# disk.dm-0.out‚60681.60986328125$ disk.nvme0c0n1.in‚44007.44140625( disk.nvme0c0n1.out‚60681.60986328125 network.recv‚ 7.461174e+06) disk./.usagePercent‚32.76781337288329 disk.sda3.out‚ 73.1171875 disk.sda.out‚ 73.1171875 disk.sda1.out‚0 disk.sda2.in‚0$ memory_percent‚68.25826168060303! proc.memory.rssMB‚ 2299.671875% disk./.usageGB‚488.26409912109375 disk.sda3.in‚ 37233.546875* proc.memory.percent‚14.036083221435547 proc.cpu.threads‚17& disk.nvme0n1.out‚60681.60986328125 disk.drbd2165.in‚0 disk.drbd2165.out‚0 disk.sda.in‚ 37233.546875 disk.sda1.in‚0 disk.sda2.out‚0) proc.memory.availableMB‚ 5200.56640625 cpu‚27.229422615779594 network.sent‚ 1.9423576e+07" disk.nvme0n1.in‚44007.44140625 disk.dm-1.in‚ 37233.546875 disk.dm-1.out‚ 73.1171875 disk.dm-0.in‚44007.44140625›þil2¢#- _wandbruntime‚3977  _runtime‚3977ù´›µÊ£#Ä _step‚1229 ' lr-AdamW/pg1‚0.00023265960908447448 ' lr-AdamW/pg2‚0.00023265960908447448 trainer/global_step‚14949 % _timestamp‚1.7866626288048475e+09 _runtime‚ 3977.33487056![=¨Ï¤#É ' lr-AdamW/pg1‚0.00023265960908447448 ' lr-AdamW/pg2‚0.00023265960908447448 trainer/global_step‚14949 % _timestamp‚1.7866626288048475e+09 _runtime‚ 3977.33487056 _step‚1229Í `?v2¥#- _wandbruntime‚3977  _runtime‚3977Ê.¶Å'¦#¿O GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 D;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚5264 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚359468226632704 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚50188727 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚4287 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚55361993 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 4552982528 UHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚64681984 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚41348020 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚322 M@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚59049984 BDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4876 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚407994224410624 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚407994207305728 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 4552982528 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4289 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 KDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚60 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 3607568326189 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚55361991 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚12543627 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 845178880 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚409188553236013 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚262 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚34468305 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 3615136342016 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 PCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚58387380 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚262 _step‚1230 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚405581001949184 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 5368709120 F>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 125829120 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 125829120 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚47011932 PIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚73 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚359468161950720 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚4289 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 4654602240 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚ 105550720 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 122306526 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚50184440 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚429 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚12543700 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 831258624 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚407994224410624 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 125829120 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚411609301702656 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚ 105546431 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚50188727 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 3615136342016 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚55361993 JBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚665 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 trainer/global_step‚14949 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚411609360752640 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 5368709120 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚44038144 F?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚60 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚407994207305728 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚61478912 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚322 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 3607609674209 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17104896 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 122905600 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚41945088 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚209 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 5494538240 KBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚4876 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚41945088 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚411609301702656 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 122905600 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚20643840 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 3823023875072 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 @7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚5264 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 4654602240 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17104896 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚55361991 IBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚75 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚223 IBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚60 JDeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4289 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 KDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚60 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 3607568326189 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚55361991 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚12543627 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 845178880 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚409188553236013 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚262 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚34468305 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 3615136342016 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 PCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚58387380 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚262 _step‚1230 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚405581001949184 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 5368709120 F>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 125829120 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 125829120 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚47011932 PIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚73 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚359468161950720 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚4289 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 4654602240 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚ 105550720 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 122306526 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚50184440 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚429 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚12543700 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 831258624 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚407994224410624 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 125829120 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚411609301702656 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚ 105546431 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚50188727 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 3615136342016 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚55361993 JBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚665 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 trainer/global_step‚14949 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚411609360752640 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 5368709120 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚44038144 F?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚60 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚407994207305728 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚61478912 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚322 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 3607609674209 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17104896 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 122905600 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚41945088 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚209 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 5494538240 KBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚4876 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚41945088 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚411609301702656 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 122905600 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚20643840 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 3823023875072 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 @7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚5264 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 4654602240 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17104896 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚55361991 IBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚75 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚223 IBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚60 JDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4876Î âËO2¨#- _wandbruntime‚3977  _runtime‚3977™r©©#ÈM NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚55365546 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚4293 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 4654602240 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚262 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚ 105557566 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚41948160 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 3823316414464 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚50326528 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 125829120 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚262 _runtime‚ 3977.50180102 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚411635914162176 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚47015033 SFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚70970368 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚411635855109120 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654602240 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 4552982528 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚359491451536384 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚55365546 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚34470510 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚223 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚4295 E5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5264 PDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521206 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 5494538240 % _timestamp‚1.7866626289714303e+09 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚20643840 NADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚58387428 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚405607170703360 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 5368709120 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚322 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚408020529643520 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 3615325465600 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 122905600 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 3607798898801 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚50187727 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚411635914162176 NADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚59053056 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 3615325465600 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚409214969602161 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚41348068 B9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚5264 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚34470512 NGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚76 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 845178880 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚61478912 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚322 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 125829120 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚12544445 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 K>DeviceStatsMonitor.on_train_batchäƒM"À_end/active_bytes.all.current‚59053056 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 122306526 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚322 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 3615367413760 G@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚78 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚355668185448448 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 3823266087936 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 trainer/global_step‚14949 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚408020546748416 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 831258624 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚50192020 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚408020546748416 H@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚665 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 3615367413760 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚50187727 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚429 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 5368709120 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚ 105553271 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 4552982528 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 122905600 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚55365544 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚50192020 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚355668206092288 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17104896 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 105557566 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚4293 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 5494538240 I@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚4876 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 125829120 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚55365544 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚47014955 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚409215027989589 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚405607187742720 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 _step‚1231 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚262 IBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚60 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚41948160 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 4517085184 G@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚60 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚ 105553271 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17104896 BDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654602240 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 4552982528 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚359491451536384 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚55365546 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚34470510 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚223 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚4295 E5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5264 PDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521206 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 5494538240 % _timestamp‚1.7866626289714303e+09 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚20643840 NADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚58387428 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚405607170703360 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 5368709120 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚322 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚408020529643520 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 3615325465600 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 122905600 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 3607798898801 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚50187727 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚411635914162176 NADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚59053056 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 3615325465600 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚409214969602161 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚41348068 B9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚5264 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚34470512 NGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚76 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 845178880 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚61478912 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚322 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 125829120 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚12544445 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 K>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59053056 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 122306526 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚322 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 3615367413760 G@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚78 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚355668185448448 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 3823266087936 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 trainer/global_step‚14949 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚408020546748416 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 831258624 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚50192020 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚408020546748416 H@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚665 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 3615367413760 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚50187727 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚429 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 5368709120 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚ 105553271 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 4552982528 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 122905600 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚55365544 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚50192020 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚355668206092288 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17104896 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 105557566 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚4293 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 5494538240 I@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚4876 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 125829120 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚55365544 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚47014955 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚409215027989589 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚405607187742720 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 _step‚1231 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚262 IBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚60 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚41948160 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 4517085184 G@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚60Ï –¼ÖB2«#- _wandbruntime‚3977  _runtime‚3977;vË—u¬#ï 1loss/FRt2/assignment_loss‚0.44560466567569124 /loss/FRt1/detection_loss‚0.5306249112764041 epoch‚37 _step‚1232 0loss/FRt1/assignment_loss‚0.5443543352700159 /loss/FRt2/detection_loss‚0.4414523054921665 &loss/total_loss‚1.9620362177142778 trainer/global_step‚14949 $ _timestamp‚1.786662628972464e+09 _runtime‚3977.502581314ištz­#ô 1loss/FRt2/assignment_loss‚0.44560466567569124 /loss/FRt1/detection_loss‚0.5306249112764041 epoch‚37 _step‚1232 0loss/FRt1/assignment_loss‚0.5443543352700159 /loss/FRt2/detection_loss‚0.4414523054921665 &loss/total_loss‚1.9620362177142778 trainer/global_step‚14949 $ _timestamp‚1.786662628972464e+09 _runtime‚3977.502581314Ð ‹Ð°¨2®#- _wandbruntime‚3989  _runtime‚3989K‰nñʯ#Ä _step‚1233 ' lr-AdamW/pg1‚0.00022832444940148797 ' lr-AdamW/pg2‚0.00022832444940148797 trainer/global_step‚14999 % _timestamp‚1.7866626409624662e+09 _runtime‚ 3989.49247587,ðÜÏ°#É ' lr-AdamW/pg2‚0.00022832444940148797 trainer/global_step‚14999 % _timestamp‚1.7866626409624662e+09 _runtime‚ 3989.49247587 _step‚1233 ' lr-AdamW/pg1‚0.00022832444940148797Ñ  ‚‹2±#- _wandbruntime‚3989  _runtime‚3989U ‹.Å'²#¿O KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚47163274 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 5368709120 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚209 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚410509463110801 _runtime‚3989.496875386 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚50349542 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚360632801698816 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 3619171484817 KDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚60 trainer/global_step‚14999 PIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚66 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚61478912 JDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4876 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚412937064408064 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚12584651 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚34578555 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 4552982528 IBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚68 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚360632866380800 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 IBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚60 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 3626681162752 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚ 105893472 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚55539641 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 3835294983168 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚410509404723421 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 3626723107840 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚34578557 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 125829120 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚55539643 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚409310341300224 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚409310324195328 UHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚64681984 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 845178880 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 125829120 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚50353829 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚409310341300224 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 3626723107840 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 _step‚1234 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 5368709120 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚412937064408064 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 4552982528 PCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚58387380 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚41348020 C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚4289 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 122905600 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚50353829 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚20643840 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 5494538240 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 4654602240 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚4287 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚4287 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚322 M@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚59049984 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚41945088 PCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚59049984 KBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚4876 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚12584717 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 BDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚412937005358080 D;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚5264 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚47163206 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚406890291625984 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚406890274586624 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚409310324195328 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 831258624 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 5494538240 % _timestamp‚1.7866626409633765e+09 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4289 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 3619130136797 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 F?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚60 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚356797527359488 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 125829120 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17104896 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 3835339021312 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚262 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 @7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚5264 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚55539641 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚50349542 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 122905600춲ô=³#ÄO G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4289 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 3619130136797 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 F?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚60 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚356797527359488 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 125829120 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17104896 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 3835339021312 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚262 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 @7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚5264 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚55539641 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚50349542 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 122905600 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚47163274 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 5368709120 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚209 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚410509463110801 _runtime‚3989.496875386 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚50349542 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚360632801698816 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 3619171484817 KDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚60 trainer/global_step‚14999 PIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚66 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚61478912 JDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4876 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚412937064408064 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚12584651 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚34578555 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 4552982528 IBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚68 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚360632866380800 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 IBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚60 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 3626681162752 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚ 105893472 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚55539641 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 3835294983168 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚410509404723421 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 3626723107840 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚34578557 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 125829120 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚55539643 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚409310341300224 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚409310324195328 UHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚64681984 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 845178880 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 125829120 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚50353829 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚409310341300224 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 3626723107840 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 _step‚1234 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 5368709120 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚412937064408064 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 4552982528 PCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚58387380 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚41348020 C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚4289 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 122905600 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚50353829 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚20643840 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 5494538240 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 4654602240 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚4287 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚4287 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 Lù±4¼ FDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚322 M@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚59049984 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚41945088 PCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚59049984 KBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚4876 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚12584717 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 BDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚412937005358080 D;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚5264 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚47163206 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚406890291625984 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚406890274586624 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚409310324195328 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 831258624 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 5494538240 % _timestamp‚1.7866626409633765e+09Ò 9)T2´#- _wandbruntime‚3989  _runtime‚3989~XaòÏ&µ#ÉM WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚412963617817600 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚12585467 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521206 EDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 105900318 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚50357122 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚322 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚410535879476997 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 3619360709409 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚429 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚34580760 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚4295 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚55543196 K>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59053056 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 5494538240 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 5368709120 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚409336663638016 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚55543196 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654602240 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 trainer/global_step‚14999 NADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚59053056 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚50352829 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚223 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚406916460380160 _runtime‚3989.659195937 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 3626954179584 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 4552982528 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 5368709120 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚412963558764544 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚409336646533120 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 845178880 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 5494538240 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚262 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚262 G@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚60 >5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5264 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚41948160 H@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚665 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 4517085184 B9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚5264 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 125829120 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚322 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚360656159469568 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 3835584737280 BDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚47166227 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚360656094790656 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚322 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚209 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 125829120 _step‚1235 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚ 105900318 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 4654602240 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 3626954179584 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 122905600 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚44035072 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚4293 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17104896 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚409336646533120 DDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 3626912231424 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚12585532 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚322 D=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚60 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚406916477419520 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚ 105896023 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚55543194 EDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 3626912231424 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚12585532 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚322 D=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚60 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚406916477419520 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚ 105896023 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚55543194 EDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521206 EDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 105900318 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚50357122 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚322 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚410535879476997 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 3619360709409 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚429 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚34580760 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚4295 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚55543196 K>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59053056 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 5494538240 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 5368709120 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚409336663638016 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚55543196 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654602240 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 trainer/global_step‚14999 NADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚59053056 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚50352829 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚223 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚406916460380160 _runtime‚3989.659195937 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 3626954179584 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 4552982528 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 5368709120 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚412963558764544 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚409336646533120 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 845178880 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 5494538240 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚262 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚262 G@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚60 >5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5264 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚41948160 H@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚665 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 4517085184 B9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚5264 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 125829120 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚322 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚360656159469568 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 3835584737280 BDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚47166227 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚360656094790656 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚322 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚209 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 125829120 _step‚1235 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚ 105900318 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 4654602240 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 3626954179584 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 122905600 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚44035072 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚4293 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17104896Ó ûN°2·#- _wandbruntime‚3989  _runtime‚3989¨Áv¸#ð $ _timestamp‚1.786662641129838e+09 _runtime‚3989.659742059 0loss/FRt1/assignment_loss‚0.5359381356996493 1loss/FRt2/assignment_loss‚0.40488792007563235 /loss/FRt1/detection_loss‚0.5131838248844459 0loss/FRt2/detection_loss‚0.49063884230519755 _step‚1236 &loss/total_loss‚1.9446487229649252 epoch‚37 trainer/global_step‚14999i0Tä{¹#õ &loss/total_loss‚1.9446487229649252 epoch‚37 trainer/global_step‚14999 $ _timestamp‚1.786662641129838e+09 _runtime‚3989.659742059 0loss/FRt1/assignment_loss‚0.5359381356996493 1loss/FRt2/assignment_loss‚0.40488792007563235 /loss/FRt1/detection_loss‚0.5131838248844459 0loss/FRt2/detection_loss‚0.49063884230519755 _step‚1236Ô qU0Û”º#‚(:‰ ñùÓůöà gpu.0.gpu‚60.0 gpu.0.memory‚11, gpu.0.memoryAllocated‚7.0313262939453125) gpu.0.memoryAllocatedBytes‚ 6039863296 gpu.0.temp‚35.0 gpu.0.powerWatts‚127.32' gpu.0.enforcedPowerLimitWatts‚500.0 gpu.0.powerPercent‚25.464 gpu.0.smClock‚1410 gpu.0.memoryClock‚1593$ gpu.0.correctedMemoryErrors‚1955# gpu.0.uncorrectedMemoryErrors‚0lØG9—»#‚(:Œ ñùÓ†ä³ã* proc.memory.percent‚14.036083221435547 cpu‚27.229118152165434 proc.cpu.threads‚17 disk.dm-0.in‚ 44014.2109375 disk.sda3.in‚ 37233.546875# disk.nvme0c0n1.in‚ 44014.2109375 disk.sda.out‚ 73.39453125 disk.sda2.out‚0$ memory_percent‚68.25659275054932 disk.dm-1.in‚ 37233.546875 disk.drbd2165.in‚0 disk.drbd2165.out‚0 disk.sda3.out‚ 73.39453125( disk.nvme0c0n1.out‚60685.45263671875 disk.sda1.out‚0 disk.sda2.in‚0! disk.nvme0n1.in‚ 44014.2109375 network.sent‚ 1.9496642e+07 network.recv‚ 7.491472e+06) disk./.usagePercent‚32.76780492463466# disk.dm-0.out‚60685.45263671875 disk.sda.in‚ 37233.546875 disk.sda1.in‚0& disk.nvme0n1.out‚60685.45263671875! proc.memory.rssMB‚ 2299.671875# disk./.usageGB‚488.263973236084 disk.dm-1.out‚ 73.39453125) proc.memory.availableMB‚ 5200.839843757P1Á2¼#- _wandbruntime‚4001  _runtime‚4001 <—‡Ê½#Ä ' lr-AdamW/pg2‚0.00022402279596428636 trainer/global_step‚15049 $ _timestamp‚1.786662653096063e+09 _runtime‚4001.626049107 _step‚1237 ' lr-AdamW/pg1‚0.00022402279596428636!=kϾ#É ' lr-AdamW/pg1‚0.00022402279596428636 ' lr-AdamW/pg2‚0.00022402279596428636 trainer/global_step‚15049 $ _timestamp‚1.786662653096063e+09 _runtime‚4001.626049107 _step‚1237Õ C'¨‹2¿#- _wandbruntime‚4001  _runtime‚4001XnzñÀ#¿O F8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚ 106231935 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 4552982528 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 5368709120 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚322 KDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚60 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚414264709013504 D;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚5264 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 5368709120 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 JDeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4289 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚50518931 PCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚58387380 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚4287 KBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚4876 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚20643840 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 125829120 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚262 F?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚60 M@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚59049984 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 122905600 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚12625341 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 125829120 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚41945088 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 3638267928576 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚34688807 UHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚66779136 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚411830314598209 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚429 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 5494538240 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚4289 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚ 106236224 BDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚361797510839808 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚357949895999488 _runtime‚4001.630862667 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17104896 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 122905600 JDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚414264709013504 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚361797444060672 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 3847614840320 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚61478912 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 _step‚1238 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚55717291 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚34688805 PIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚59 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 3638309873664 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚414264768063488 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 3847568705024 @7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚5264 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17104896 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 4654602240 JBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚665 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 831258624 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚50514644 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚410626458189824 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 5368709120 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚4287 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚47314148 IBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚61 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚408199581302784 % _timestamp‚1.7866626530969303e+09 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 4654602240 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚357949875355648 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚408199564263424 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 G>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4876 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚50514644 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚12625282 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 trainer/global_step‚15049ÞQEµÊ'Á#ÄO MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 IBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚60 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚414264709013504 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚361797444060672 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 3847614840320 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚61478912 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 _step‚1238 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚55717291 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚34688805 PIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚59 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 3638309873664 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚414264768063488 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 3847568705024 @7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚5264 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17104896 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 4654602240 JBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚665 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 831258624 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚50514644 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚410626458189824 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 5368709120 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚4287 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚47314148 IBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚61 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚408199581302784 % _timestamp‚1.7866626530969303e+09 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 4654602240 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚357949875355648 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚408199564263424 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 G>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4876 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚50514644 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚12625282 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 trainer/global_step‚15049 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚ 106231935 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 4552982528 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 5368709120 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚322 KDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚60 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚414264709013504 D;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚5264 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 5368709120 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 JDeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4289 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚50518931 PCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚58387380 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚4287 KBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚4876 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚20643840 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 125829120 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚262 F?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚60 M@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚59049984 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 122905600 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚12625341 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 125829120 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚41945088 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 3638267928576 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚34688807 UHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚66779136 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚411830314598209 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚429 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 5494538240 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚4289 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚ 106236224 BDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚361797510839808 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚357949895999488 _runtime‚4001.630862667 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17104896 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 122905600 J5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5264 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 4552982528 _runtime‚4001.790426433 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚55720846 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚46132224 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚223 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 125829120 IBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚60 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚41948160 H@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚665 G@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚64 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚47317121 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚34691012 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 5368709120 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚410652780527616 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 4552982528 EDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 3638540945408 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚411856730964405 G@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚60 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚55720844 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚50522224 NADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚59053056 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚357972922728448 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚50517931 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚410652763422720 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚50522224 I@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚4876 SFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚66776064 P6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚322 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 3630922520017 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚322 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚12626111 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 122306526 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 5368709120 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 125829120 _step‚1239 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 3638540945408 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚55720846 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚ 106243070 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚4293 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 3638498997248 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚ 106238775 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚50517931 NADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚58387428 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚408225767096320 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚4295 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17104896 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚429 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚361820737185280 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 831258624 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 3847814456832 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 EDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 106243070 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 125829120 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 5368709120 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654602240 BDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 % _timestamp‚1.7866626532601619e+09 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚414291321473024 K>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59053056 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 DDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521206 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚408225750056960 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 5494538240 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚ 106238775 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚410652763422720 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚414291321473024 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 3638498997248 B9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚5264 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚322 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚61478912 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 4517085184 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚262 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚414291262419968 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚4293 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚12626173 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚209 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 D=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚60 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚411856672576977Ä/ùV>$Ä#ÎM J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚55720844 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚50522224 NADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚59053056 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚357972922728448 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚50517931 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚410652763422720 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚50522224 I@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚4876 SFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚66776064 P6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚322 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 3630922520017 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚322 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚12626111 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 122306526 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 5368709120 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 125829120 _step‚1239 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 3638540945408 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚55720846 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚ 106243070 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚4293 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 3638498997248 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚ 106238775 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚50517931 NADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚58387428 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚408225767096320 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚4295 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17104896 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚429 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚361820737185280 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 831258624 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 3847814456832 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 EDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 106243070 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 125829120 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 5368709120 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654602240 BDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 % _timestamp‚1.7866626532601619e+09 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚414291321473024 K>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59053056 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 DDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521206 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚408225750056960 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 5494538240 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚ 106238775 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚410652763422720 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚414291321473024 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 3638498997248 B9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚5264 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚322 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚61478912 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 4517085184 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚262 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚414291262419968 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚4293 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚12626173 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚209 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 D=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚60 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚411856672576977 trainer/global_step‚15049 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚410652780527616 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚361820803961344 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 845178880 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚34691010 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚262 >5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5264 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 4552982528 _runtime‚4001.790426433 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚55720846 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚46132224 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚223 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 125829120 IBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚60 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚41948160 H@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚665 G@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚64 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚47317121 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚34691012 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 5368709120 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚410652780527616 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 4552982528 EDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 3638540945408 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚411856730964405 G@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚60× Ë S’2Å#- _wandbruntime‚4001  _runtime‚4001¡}`„uÆ#ï 0loss/FRt1/assignment_loss‚0.5119468751636238 0loss/FRt2/assignment_loss‚0.4474693828785397 0loss/FRt1/detection_loss‚0.46554237395022724 /loss/FRt2/detection_loss‚0.4703431949065669 &loss/total_loss‚1.8953018268989577 epoch‚37 trainer/global_step‚15049 $ _timestamp‚1.786662653261252e+09 _runtime‚4001.791091521 _step‚1240ÂÒ6DzÇ#ô 0loss/FRt1/assignment_loss‚0.5119468751636238 0loss/FRt2/assignment_loss‚0.4474693828785397 0loss/FRt1/detection_loss‚0.46554237395022724 /loss/FRt2/detection_loss‚0.4703431949065669 &loss/total_loss‚1.8953018268989577 epoch‚37 trainer/global_step‚15049 $ _timestamp‚1.786662653261252e+09 _runtime‚4001.791091521 _step‚1240Ø KèªÁ–È#‚(:‹ €žùÓ€ž§á gpu.0.gpu‚64.0 gpu.0.memory‚12, gpu.0.memoryAllocated‚7.0313262939453125) gpu.0.memoryAllocatedBytes‚ 6039863296 gpu.0.temp‚35.0 gpu.0.powerWatts‚123.994' gpu.0.enforcedPowerLimitWatts‚500.0 gpu.0.powerPercent‚24.7988 gpu.0.smClock‚1410 gpu.0.memoryClock‚1593$ gpu.0.correctedMemoryErrors‚1955# gpu.0.uncorrectedMemoryErrors‚0ÇÚ˜½’É#‚(:‡ €žùÓ´²¹ã% disk./.usageGB‚487.13715744018555 disk.sda.in‚ 37233.546875 disk.sda.out‚ 73.55078125 disk.sda1.out‚0 disk.sda2.in‚0% disk.nvme0n1.out‚60691.5556640625 disk.dm-1.in‚ 37233.546875) proc.memory.availableMB‚ 5200.50390625 network.recv‚ 7.524135e+06" disk.nvme0c0n1.in‚ 44022.390625' disk.nvme0c0n1.out‚60691.5556640625 disk.dm-1.out‚ 73.55078125* proc.memory.percent‚14.036083221435547 network.sent‚ 1.959229e+07 disk.sda3.out‚ 73.55078125 disk.sda2.out‚0 disk.nvme0n1.in‚ 44022.390625 disk.dm-0.in‚ 44022.390625$ memory_percent‚68.25864315032959! proc.memory.rssMB‚ 2299.671875 cpu‚27.228761966681553) disk./.usagePercent‚32.69218337111049 disk.sda3.in‚ 37233.546875 disk.sda1.in‚0 disk.drbd2165.in‚0 disk.drbd2165.out‚0" disk.dm-0.out‚60691.5556640625 proc.cpu.threads‚17a1ÿô2Ê#- _wandbruntime‚4014  _runtime‚4014Å2ÒMÉË#à & lr-AdamW/pg1‚0.0002197549250600315 & lr-AdamW/pg2‚0.0002197549250600315 trainer/global_step‚15099 % _timestamp‚1.7866626654773726e+09 _runtime‚4014.007357938 _step‚1241UéׯÎÌ#È & lr-AdamW/pg1‚0.0002197549250600315 & lr-AdamW/pg2‚0.0002197549250600315 trainer/global_step‚15099 % _timestamp‚1.7866626654773726e+09 _runtime‚4014.007357938 _step‚1241Ù šð•î2Í#- _wandbruntime‚4014  _runtime‚4014ð5%kÅ'Î#¿O TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚50684033 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚413151166085617 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚262 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚50679746 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚20643840 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚409508853940224 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚41348020 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 5368709120 % _timestamp‚1.7866626654782848e+09 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 4654602240 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚209 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 4517085184 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 3649854694400 IBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚62 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 845178880 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 5368709120 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 5368709120 IBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚60 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚322 F?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚60 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚413151107698237 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚ 106574687 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 4654602240 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 3649896639488 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚4287 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚12666227 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 831258624 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚50684033 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17104896 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚41945088 JDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚415592412668928 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚41945088 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 5494538240 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 3649896639488 D;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚5264 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚50679746 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 125829120 G>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4876 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚12666167 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚411942557974528 KBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚4876 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚415592471718912 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚411942557974528 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 125829120 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚322 trainer/global_step‚15099 PCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚58387380 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 122306526 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚55894941 F>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚322 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚429 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 4552982528 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚55894943 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 3859848022528 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 3642295106033 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚322 _step‚1242 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4289 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚47465284 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚47465222 @7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚5264 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 122905600 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 3859887866368 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚61478912 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 122905600 PIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚60 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 3642253758013 JDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚415592412668928 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚41945088 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 5494538240 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 3649896639488 D;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚5264 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚50679746 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 125829120 G>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4876 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚12666167 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚411942557974528 KBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚4876 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚415592471718912 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚411942557974528 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 125829120 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚322 trainer/global_step‚15099 PCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚58387380 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 122306526 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚55894941 F>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚322 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚429 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 4552982528 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚55894943 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 3859848022528 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 3642295106033 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚322 _step‚1242 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4289 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚47465284 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚47465222 @7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚5264 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 122905600 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 3859887866368 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚61478912 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 122905600 PIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚60 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 3642253758013 JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654602240 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚55898494 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 4517085184 G@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚60 H@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚665 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚61478912 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 5494538240 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 >5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5264 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚12667014 SFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚64678912 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚55898494 B9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚5264 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚55898496 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 5494538240 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚411968897417216 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚322 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 3642484330625 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚50683033 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 5368709120 NADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚58387428 IBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚60 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 3650127711232 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 106585822 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚4293 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 3860134794752 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 3650085763072 D=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚60 _step‚1243 PDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚ 106585822 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚4295 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 3650127711232 NGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚70 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 5368709120 EDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59053056 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚50687326 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚429 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚223 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚413177582451813 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚55898496 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 3642525678693 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 122306526 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚ 106581527 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚415619025128448 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 4654602240 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚411968880312320 G@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚72 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521206 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚41348068 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚41948160 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚44035072 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 125829120 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚322 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚4293 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚41948160 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 122905600 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚34801262 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚362985446807040 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 BDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚47468274 I@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚4876 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17104896 E6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚322 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚415618966075392 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚50683033 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚359125312012288Â/AÉÔ&Ò#ÎM J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚50683033 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 5368709120 NADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚58387428 IBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚60 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 3650127711232 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 106585822 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚4293 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 3860134794752 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 3650085763072 D=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚60 _step‚1243 PDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚ 106585822 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚4295 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 3650127711232 NGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚70 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 5368709120 EDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59053056 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚50687326 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚429 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚223 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚413177582451813 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚55898496 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 3642525678693 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 122306526 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚ 106581527 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚415619025128448 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 4654602240 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚411968880312320 G@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚72 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521206 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚41348068 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚41948160 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚44035072 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 125829120 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚322 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚4293 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚41948160 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 122905600 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚34801262 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚362985446807040 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 BDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚47468274 I@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚4876 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17104896 E6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚322 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚415618966075392 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚50683033 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚359125312012288 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 DDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654602240 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚55898494 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 4517085184 G@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚60 H@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚665 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚61478912 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 5494538240 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 >5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5264 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚12667014 SFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚64678912 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚55898494 B9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚5264 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚55898496 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 5494538240 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚411968897417216 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚322 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 3642484330625Û ¯ÿB2Ó#- _wandbruntime‚4014  _runtime‚4014zù¼tÔ#î 0loss/FRt1/assignment_loss‚0.5238905987777269 0loss/FRt2/assignment_loss‚0.3919328547144011 /loss/FRt1/detection_loss‚0.5005759750418312 /loss/FRt2/detection_loss‚0.4856123957995159 %loss/total_loss‚1.902011824333475 epoch‚37 trainer/global_step‚15099 % _timestamp‚1.7866626656437597e+09 _runtime‚4014.173573647 _step‚1244$åJyÕ#ó _runtime‚4014.173573647 _step‚1244 0loss/FRt1/assignment_loss‚0.5238905987777269 0loss/FRt2/assignment_loss‚0.3919328547144011 /loss/FRt1/detection_loss‚0.5005759750418312 /loss/FRt2/detection_loss‚0.4856123957995159 %loss/total_loss‚1.902011824333475 epoch‚37 trainer/global_step‚15099 % _timestamp‚1.7866626656437597e+09Ü åOëO¡Ö#‚(:– žùÓ•žŒá gpu.0.gpu‚64.0 gpu.0.memory‚12, gpu.0.memoryAllocated‚7.0313262939453125) gpu.0.memoryAllocatedBytes‚ 6039863296 gpu.0.temp‚36.0 gpu.0.powerWatts‚181.041' gpu.0.enforcedPowerLimitWatts‚500.0) gpu.0.powerPercent‚36.208200000000005 gpu.0.smClock‚1410 gpu.0.memoryClock‚1593$ gpu.0.correctedMemoryErrors‚1955# gpu.0.uncorrectedMemoryErrors‚0¹üli‘×#‚(:† žùÓÑÿ’ã disk.sda2.in‚0 disk.drbd2165.in‚0 disk.dm-0.in‚ 44022.390625 disk.sda3.in‚ 37233.546875 disk.sda3.out‚ 73.7421875$ memory_percent‚68.25861930847168 disk.nvme0n1.in‚ 44022.390625 network.sent‚ 1.9660823e+07) disk./.usagePercent‚32.69218541917076$ disk./.usageGB‚487.1371879577637" disk.nvme0c0n1.in‚ 44022.390625( disk.nvme0c0n1.out‚60695.22119140625 disk.sda.in‚ 37233.546875 disk.drbd2165.out‚0 network.recv‚ 7.549324e+06& disk.nvme0n1.out‚60695.22119140625# disk.dm-0.out‚60695.22119140625! proc.memory.rssMB‚ 2299.671875 disk.sda1.out‚0 disk.sda2.out‚0 disk.dm-1.in‚ 37233.546875 disk.dm-1.out‚ 73.7421875( proc.memory.availableMB‚ 5200.5078125* proc.memory.percent‚14.036083221435547 cpu‚27.228290064151867 proc.cpu.threads‚17 disk.sda.out‚ 73.7421875 disk.sda1.in‚0ûsrÝ2Ø#- _wandbruntime‚4026  _runtime‚4026Î’hÆÙ#À % lr-AdamW/pg1‚0.000215521110806096 % lr-AdamW/pg2‚0.000215521110806096 trainer/global_step‚15149 $ _timestamp‚1.786662677999081e+09 _runtime‚4026.529099213 _step‚1245[Ö‘CËÚ#Å % lr-AdamW/pg1‚0.000215521110806096 % lr-AdamW/pg2‚0.000215521110806096 trainer/global_step‚15149 $ _timestamp‚1.786662677999081e+09 _runtime‚4026.529099213 _step‚1245Ý ë—2Û#- _wandbruntime‚4026  _runtime‚4026ÉhRÅ'Ü#¿O PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚50844848 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚12707015 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚410818143617024 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 122306526 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚50844848 M@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚59049984 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 122905600 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚416920116324352 D;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚5264 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚56072593 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚429 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚414471959185645 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 3653815568621 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 5368709120 IBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚60 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 4654602240 KBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚4876 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 5494538240 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚413258691969024 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚413258674864128 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚209 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚360254612635648 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚322 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚50849135 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 3872152595968 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 PCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚58387380 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚262 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚4287 PIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚71 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚322 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 5368709120 _runtime‚4026.533870457 JDeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4289 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 3872106460672 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚364126785875456 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 125829120 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 4552982528 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 3661483405312 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 5368709120 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 125829120 F>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚41945088 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚46135296 BDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚416920116324352 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 4654602240 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚56072591 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 3661483405312 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚12707086 JBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚665 UHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚66779136 G>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4876 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 122905600 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 3653856916641 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚50849135 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚416920175374336 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚47616393 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚322 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚ 106917439 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚ 106921728 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚223 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚416920175374336 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚20643840 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 5494538240 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚4287 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚364126719096320 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚262 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚41945088 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 831258624 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚41348020 % _timestamp‚1.7866626780000067e+09 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 3661441460224 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 3661441460224 IBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚73 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 4628521206 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚61478912 F?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚60 _step‚1246 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚413258674864128 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚262 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚322 PCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚59049984 trainer/global_step‚15149 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚56072591 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 845178880 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17104896 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚410818160656384 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚34909307 JDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚429 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚414471959185645 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 3653815568621 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 5368709120 IBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚60 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 4654602240 KBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚4876 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 5494538240 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚413258691969024 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚413258674864128 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚209 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚360254612635648 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚322 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚50849135 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 3872152595968 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 PCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚58387380 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚262 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚4287 PIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚71 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚322 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 5368709120 _runtime‚4026.533870457 JDeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4289 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 3872106460672 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚364126785875456 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 125829120 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 4552982528 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 3661483405312 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 5368709120 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 125829120 F>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚41945088 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚46135296 BDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚416920116324352 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 4654602240 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚56072591 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 3661483405312 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚12707086 JBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚665 UHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚66779136 G>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4876 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 122905600 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 3653856916641 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚50849135 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚416920175374336 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚47616393 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚322 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚ 106917439 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚ 106921728 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚223 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚416920175374336 \ODeviceStatsMonitor.on_train_batch_start/inactih7)Ðve_split_bytes.large_pool.current‚20643840 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 5494538240 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚4287 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚364126719096320 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚262 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚41945088 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 831258624 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚41348020 % _timestamp‚1.7866626780000067e+09 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 3661441460224 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 3661441460224Þ @H2Þ#- _wandbruntime‚4026  _runtime‚4026 ¤ñÏ&ß#ÉM WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚414498433939221 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 5494538240 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚4293 DDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521206 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 122905600 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚47619293 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚364150012876288 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚ 106924279 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 106928574 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚ 106924279 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚209 SFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚66776064 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 3872352867840 IBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚60 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 125829120 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚262 K>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59053056 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚413285014306816 I@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚4876 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚12707783 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 831258624 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚322 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 122905600 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 4654602240 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17104896 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 4552982528 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚413285014306816 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 3661672528896 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚20643840 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚4295 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚34911512 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚360277660008448 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 _runtime‚4026.691237429 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654602240 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚429 trainer/global_step‚15149 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 845178880 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚41348068 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 5494538240 _step‚1247 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚416946728783872 EDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚34911510 NGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚77 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚410844329410560 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 3661714477056 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 3661672528896 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚56076146 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 H@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚665 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚12707860 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 G@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚60 P5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5264 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚4293 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚416946728783872 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚41948160 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17104896 B9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚5264 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚61478912 B6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚322 D=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚60 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚413284997201920 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚56076144 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 3872399000064 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚414498375551793 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 122306526 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 5368709120 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 125829120 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚50852428 G@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚79 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 5368709120 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚50848135 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 3654046141233 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚262 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚ 106928574 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚322 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚410844346449920 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚56076144 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚413284997201920 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0T óÔ&à#ÎM >5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5264 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚4293 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚416946728783872 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚41948160 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17104896 B9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚5264 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚61478912 B6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚322 D=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚60 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚413284997201920 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚56076144 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 3872399000064 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚414498375551793 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 122306526 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 5368709120 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 125829120 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚50852428 G@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚79 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 5368709120 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚50848135 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 3654046141233 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚262 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚ 106928574 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚322 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚410844346449920 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚56076144 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚413284997201920 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚414498433939221 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 5494538240 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚4293 DDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521206 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 122905600 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚47619293 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚364150012876288 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚ 106924279 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 106928574 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚ 106924279 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚209 SFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚66776064 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 3872352867840 IBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚60 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 125829120 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚262 K>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59053056 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚413285014306816 I@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚4876 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚12707783 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 831258624 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚322 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 122905600 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 4654602240 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17104896 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 4552982528 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚413285014306816 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 3661672528896 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚20643840 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚4295 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚34911512 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚360277660008448 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 _runtime‚4026.691237429 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654602240 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚429 trainer/global_step‚15149 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 845178880 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚41348068 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 5494538240 _step‚1247 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚416946728783872 EDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚34911510 NGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚77 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚410844329410560 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 3661714477056 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 3661672528896 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚56076146 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 H@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚665 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚12707860 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 G@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚60 PDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 5368709120 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚361407001919488 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚415792810673053 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 122306526 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚322 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 4654602240 PCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚59049984 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 122905600 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚47767554 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚209 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17104896 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 125829120 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚56250241 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚41945088 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚361406981275648 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚322 _step‚1250 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚51009950 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚418247879029760 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚365291375994368 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚56250243 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚56250241 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 5494538240 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 5494538240 # _timestamp‚1.78666269020175e+09 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚418247819979776 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 3673070171136 JDeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4289 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚4287 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚12747999 F?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚60 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 5494538240 PIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚67 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 845178880 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚412127450333184 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚12748066 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 PCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚58387380 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 5368709120 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚51014237 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚ 107260191 JDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4876 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚414574791753728 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 4552982528 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 3673028226048 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17104896 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚41945088 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚ 107264480 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚414574808858624 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 125829120 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 3884394718720 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚41348020 F>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚414574808858624 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚414574791753728 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 3673028226048 LDDeviceS‡ºötatsMonitor.on_train_batch_start/segment.large_pool.allocated‚262 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 4552982528 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚412127433293824 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 3665377379229 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 KBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚4876 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚47767623 UHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚66779136 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 4517085184 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 3665418727249 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚365291442773504 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 3884440854016 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚46135296 KDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚60 _runtime‚4038.734875215XǧÀÈ'ë#ÂO # _timestamp‚1.78666269020175e+09 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚418247819979776 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 3673070171136 JDeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4289 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚4287 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚12747999 F?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚60 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 5494538240 PIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚67 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 845178880 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚412127450333184 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚12748066 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 PCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚58387380 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 5368709120 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚51014237 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚ 107260191 JDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4876 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚414574791753728 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 4552982528 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 3673028226048 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17104896 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚41945088 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚ 107264480 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚414574808858624 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 125829120 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 3884394718720 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚41348020 F>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚414574808858624 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚414574791753728 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 3673028226048 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚262 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 4552982528 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚412127433293824 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 3665377379229 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 KBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚4876 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚47767623 UHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚66779136 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 4517085184 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 3665418727249 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚365291442773504 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 3884440854016 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚46135296 KDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚60 _runtime‚4038.734875215 @7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚5264 IBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚69 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 BDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 5368709120 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚361407001919488 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚415792810673053 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 122306526 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚322 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 4654602240 PCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚59049984 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 122905600 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚47767554 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚209 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17104896 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 125829120 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚56250241 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚41945088 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚361406981275648 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚322 _step‚1250 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚51009950 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚418247879029760 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚365291375994368 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚56250243 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚56250241 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 5494538240 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 5494538240â s±‡2ì#- _wandbruntime‚4038  _runtime‚4038ŠÝ‰ªÎ&í#ÈM J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚56253794 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 845178880 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 122306526 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 5368709120 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 125829120 _runtime‚4038.893429212 _step‚1251 EDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚56253796 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚51017530 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 3665607951841 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚41948160 D=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚60 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17104896 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚12748800 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚322 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚412153619087360 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚322 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚262 K>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59053056 NADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚59053056 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654602240 EDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 107271326 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 G@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚60 B9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚5264 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚418274432439296 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 5494538240 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚41948160 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚35021760 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚61478912 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚223 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521206 >5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5264 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚414601131196416 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17104896 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 NADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚58387428 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 5368709120 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚47770640 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚414601131196416 SFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚75164672 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚415819227039201 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 3884687782400 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚415819285426629 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 $ _timestamp‚1.786662690363154e+09 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 3673301242880 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚262 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚262 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚414601114091520 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚4295 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 122905600 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚4293 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚365314737074688 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚322 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚429 IBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚60 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚20643840 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 D6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚322 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚35021762 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚ 107267031 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 3673259294720 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 5494538240 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 125829120 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚ 107271326 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚56253796 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚51017530 H@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚665 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 125829120 I@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚4876 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 4654602240 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚51013237 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚365314661910016 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚412153636126720 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚418274432439296 G@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚80 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚12748878 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚209 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚361430049292288 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚4293 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚47770560 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 trainer/global_step‚15199 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚56253794 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚54520832 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 3665649299909 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 3673259294720 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚ 107267031 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 5494538240žâôÓ&î#ÍM ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚20643840 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 D6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚322 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚35021762 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚ 107267031 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 3673259294720 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 5494538240 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 125829120 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚ 107271326 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚56253796 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚51017530 H@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚665 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 125829120 I@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚4876 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 4654602240 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚51013237 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚365314661910016 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚412153636126720 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚418274432439296 G@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚80 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚12748878 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚209 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚361430049292288 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚4293 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚47770560 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 trainer/global_step‚15199 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚56253794 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚54520832 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 3665649299909 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 3673259294720 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚ 107267031 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 5494538240 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚56253794 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 845178880 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 122306526 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 5368709120 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 125829120 _runtime‚4038.893429212 _step‚1251 EDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚56253796 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚51017530 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 3665607951841 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚41948160 D=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚60 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17104896 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚12748800 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚322 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚412153619087360 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚322 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚262 K>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59053056 NADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚59053056 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654602240 EDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 107271326 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 G@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚60 B9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚5264 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚418274432439296 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 5494538240 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚41948160 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚35021760 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚61478912 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚223 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521206 >5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5264 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚414601131196416 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17104896 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 NADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚58387428 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 5368709120 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚47770640 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚414601131196416 SFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚75164672 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚415819227039201 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 3884687782400 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚415819285426629 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 $ _timestamp‚1.786662690363154e+09 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 3673301242880 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚262 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚262 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚414601114091520 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚4295 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 122905600 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚4293 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚365314737074688 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚322 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚429 IBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚60ã y(Í2ï#- _wandbruntime‚4038  _runtime‚4038«1ÄÖvð#ð 0loss/FRt1/assignment_loss‚0.5939459427299791 1loss/FRt2/assignment_loss‚0.40401335503680386 /loss/FRt1/detection_loss‚0.5154634268607917 /loss/FRt2/detection_loss‚0.4553390731979078 _runtime‚4038.894290339 _step‚1252 &loss/total_loss‚1.9687617978254823 epoch‚37 trainer/global_step‚15199 % _timestamp‚1.7866626903641815e+09®,{ñ#õ 0loss/FRt1/assignment_loss‚0.5939459427299791 1loss/FRt2/assignment_loss‚0.40401335503680386 /loss/FRt1/detection_loss‚0.5154634268607917 /loss/FRt2/detection_loss‚0.4553390731979078 _runtime‚4038.894290339 _step‚1252 &loss/total_loss‚1.9687617978254823 epoch‚37 trainer/global_step‚15199 % _timestamp‚1.7866626903641815e+09ä é~ݨ2ò#- _wandbruntime‚4043  _runtime‚4043iøó °ó#× 3Purity/0FRt/event_proportion‚0.3301943838596344 /Purity/1FRt/event_purity‚0.4709251821041107 *jet/accuracy_1_of_1‚0.3121141195297241 +validation_accuracy‚0.21738755702972412 % _timestamp‚1.7866626953452866e+09 3Purity/1FRt/event_proportion‚0.6297432780265808 -Purity/1FRt/FRt_purity‚0.4709251821041107 0Purity/2FRt/event_purity‚0.43109002709388733 trainer/global_step‚15199 2Purity/*FRtG!¾B-/event_proportion‚0.669805645942688 Purity/0FRt/event_purity‚1 -Purity/2FRt/FRt_purity‚0.5718868970870972 +particle/specificity‚0.6309453845024109 2Purity/2FRt/event_proportion‚0.040062315762043 %particle/f_score‚0.55327308177948 epoch‚37 .Purity/*FRt/FRt_purity‚0.48244917392730713 particle/accuracy_1_of_1‚1 /particle/accuracy_2_of_2‚0.6973985433578491 (particle/accuracy‚0.6338006854057312 7validation_average_jet_accuracy‚0.31752336025238037 /Purity/*FRt/event_purity‚0.4686540961265564 _runtime‚4043.875346536 +jet/accuracy_2_of_2‚0.21738755702972412 /particle/accuracy_1_of_2‚0.9253453612327576 _step‚1253 )loss/val_total_loss‚2.479045853321533 *jet/accuracy_1_of_2‚0.5864024758338928 *particle/sensitivity‚0.638947069644928,zv‚âô#Ü .Purity/*FRt/FRt_purity‚0.48244917392730713 particle/accuracy_1_of_1‚1 /particle/accuracy_2_of_2‚0.6973985433578491 (particle/accuracy‚0.6338006854057312 7validation_average_jet_accuracy‚0.31752336025238037 /Purity/*FRt/event_purity‚0.4686540961265564 _runtime‚4043.875346536 +jet/accuracy_2_of_2‚0.21738755702972412 /particle/accuracy_1_of_2‚0.9253453612327576 _step‚1253 )loss/val_total_loss‚2.479045853321533 *jet/accuracy_1_of_2‚0.5864024758338928 *particle/sensitivity‚0.638947069644928 3Purity/0FRt/event_proportion‚0.3301943838596344 /Purity/1FRt/event_purity‚0.4709251821041107 *jet/accuracy_1_of_1‚0.3121141195297241 +validation_accuracy‚0.21738755702972412 % _timestamp‚1.7866626953452866e+09 3Purity/1FRt/event_proportion‚0.6297432780265808 -Purity/1FRt/FRt_purity‚0.4709251821041107 0Purity/2FRt/event_purity‚0.43109002709388733 trainer/global_step‚15199 2Purity/*FRt/event_proportion‚0.669805645942688 Purity/0FRt/event_purity‚1 -Purity/2FRt/FRt_purity‚0.5718868970870972 +particle/specificity‚0.6309453845024109 2Purity/2FRt/event_proportion‚0.040062315762043 %particle/f_score‚0.55327308177948 epoch‚37å |oÇ·–õ#‚(:‹ ­žùÓÚä á gpu.0.gpu‚63.0 gpu.0.memory‚12, gpu.0.memoryAllocated‚7.0313262939453125) gpu.0.memoryAllocatedBytes‚ 6039863296 gpu.0.temp‚35.0 gpu.0.powerWatts‚220.913' gpu.0.enforcedPowerLimitWatts‚500.0 gpu.0.powerPercent‚44.1826 gpu.0.smClock‚1410 gpu.0.memoryClock‚1593$ gpu.0.correctedMemoryErrors‚1955# gpu.0.uncorrectedMemoryErrors‚0MÍ ‹ö#‚(:€ ­žùÓÀèÝâ disk.sda1.in‚0 disk.sda1.out‚0$ memory_percent‚68.35296154022217 proc.cpu.threads‚17' disk.nvme0c0n1.out‚60763.7509765625 network.recv‚ 7.609728e+06$ disk./.usageGB‚487.0996780395508 disk.drbd2165.in‚0" disk.dm-0.out‚60763.7509765625 disk.sda3.out‚74.21875 disk.nvme0n1.in‚ 44022.390625% disk.nvme0n1.out‚60763.7509765625" disk.nvme0c0n1.in‚ 44022.390625 disk.sda.in‚ 37233.546875 disk.sda2.out‚0 disk.dm-1.in‚ 37233.546875 disk.dm-1.out‚74.21875) proc.memory.availableMB‚ 5185.05078125# proc.memory.rssMB‚ 2315.17578125* proc.memory.percent‚14.130711555480957 disk.sda.out‚74.21875 network.sent‚ 1.9807013e+07* disk./.usagePercent‚32.689668097086575 disk.sda3.in‚ 37233.546875 disk.sda2.in‚0 cpu‚27.23340979560435 disk.drbd2165.out‚0 disk.dm-0.in‚ 44022.390625:qWª2÷#- _wandbruntime‚4056  _runtime‚4056Z ¸Ëø#Å % _timestamp‚1.7866627075185378e+09 _runtime‚4056.048560522 _step‚1254 ' lr-AdamW/pg1‚0.00020715673776423572 ' lr-AdamW/pg2‚0.00020715673776423572 trainer/global_step‚15249K­°éÐù#Ê ' lr-AdamW/pg1‚0.00020715673776423572 ' lr-AdamW/pg2‚0.00020715673776423572 trainer/global_step‚15249 % _timestamp‚1.7866627075185378e+09 _runtime‚4056.048560522 _step‚1254æ ¸…Ô2ú#- _wandbruntime‚4056  _runtime‚4056N-Ò¬Ä'û#¾O OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚413817622462464 @7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚5264 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚262 $ _timestamp‚1.786662707519473e+09 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 3688724660736 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 4654602240 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚416281845858304 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚419970587623936 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚419970528573952 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 4654602240 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 4552982528 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚35170421 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚35170419 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚41945088 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚4287 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚47968418 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 3900766178816 PCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚59049984 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 5368709120 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚262 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚223 BDeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4289 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚56488497 KBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚4876 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚362858640867328 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚417498627624402 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚429 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚366759386402304 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 3681046509958 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 3688682715648 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 _step‚1255 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚41945088 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 5494538240 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 4628521206 D;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚5264 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17104896 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚416281845858304 PIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚74 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚56488499 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚20643840 G>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4876 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 4517085184 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 3688724660736 UHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚66779136 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚322 F>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 845178880 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 IBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚60 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚51201526 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚413817639501824 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 125829120 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚262 _runtime‚4056.052869364 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚4287 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚56488499 JBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚665 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚12797923 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 3681005161938 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚ 107690025 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 3900812314112 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 125829120 trainer/global_step‚15249 y`É'ü#ÃO WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚41945088 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚4287 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚47968418 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 3900766178816 PCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚59049984 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 5368709120 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚262 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚223 BDeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4289 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚56488497 KBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚4876 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚362858640867328 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚417498627624402 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚429 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚366759386402304 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 3681046509958 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 3688682715648 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 _step‚1255 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚41945088 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 5494538240 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 4628521206 D;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚5264 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17104896 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚416281845858304 PIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚74 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚56488499 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚20643840 G>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4876 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 4517085184 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 3688724660736 UHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚66779136 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚322 F>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 845178880 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 IBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚60 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚51201526 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚413817639501824 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 125829120 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚262 _runtime‚4056.052869364 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚4287 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚56488499 JBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚665 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚12797923 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 3681005161938 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚ 107690025 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 3900812314112 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 125829120 trainer/global_step‚15249 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚413817622462464 @7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚5264 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚262 $ _timestamp‚1.786662707519473e+09 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 3688724660736 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 4654602240 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚416281845858304 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚419970587623936 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚419970528573952 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 4654602240 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 4552982528 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚35170421 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚35170419ç CDqÎ2ý#- _wandbruntime‚4056  _runtime‚4056Šaÿ; þ#ÇM K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚47971383 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 5494538240 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚262 >5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5264 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 4654602240 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚41948160 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 3688913784320 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 125829120 # _timestamp‚1.78666270768405e+09 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚56492052 G@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚75 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚51204819 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚416308185300992 NADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚59053056 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚20643840 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚61478912 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 NADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚58387428 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚419997141033472 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17104896 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚35172624 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚362881667596288 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚56492050 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 5494538240 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚262 trainer/global_step‚15249 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 4552982528 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚366782675987968 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 831258624 G@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚60 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚416308185300992 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 IBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚60 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 3901008391680 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 5368709120 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚322 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚56492050 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚12798759 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 4517085184 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 _runtime‚4056.214328965 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚416308168196096 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚47971458 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 4552982528 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654602240 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 3688913784320 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚322 E6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚322 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 3681235734550 _step‚1256 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚56492052 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 845178880 BDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521206 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 5368709120 EDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59053056 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 107696871 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚4293 I@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚4876 ^JDeviceStatsMonitor.on_train;bÝÿ_batch_end/requested_bytes.large_pool.allocated‚413843825295360 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚ 107696871 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 D8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 122306526 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚ 107692576 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚51200526 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚4293 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚419997141033472 PDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0ÁU0ÃÒ&ÿ#ÌM ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚20643840 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚61478912 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 NADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚58387428 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚419997141033472 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17104896 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚35172624 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚362881667596288 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚56492050 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 5494538240 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚262 trainer/global_step‚15249 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 4552982528 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚366782675987968 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 831258624 G@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚60 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚416308185300992 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 IBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚60 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 3901008391680 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 5368709120 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚322 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚56492050 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚12798759 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 4517085184 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 _runtime‚4056.214328965 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚416308168196096 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚47971458 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 4552982528 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654602240 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 3688913784320 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚322 E6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚322 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 3681235734550 _step‚1256 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚56492052 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 845178880 BDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521206 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 5368709120 EDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59053056 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 107696871 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚4293 I@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚4876 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚413843825295360 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚ 107696871 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 D8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 122306526 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚ 107692576 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚51200526 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚4293 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚419997141033472 PDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚47971383 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 5494538240 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚262 >5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5264 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 4654602240 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚41948160 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 3688913784320 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 125829120 # _timestamp‚1.78666270768405e+09 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚56492052 G@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚75 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚51204819 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚416308185300992 NADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚59053056è $/õ}2€$- _wandbruntime‚4056  _runtime‚4056mÝ5öv$ð _step‚1257 0loss/FRt1/assignment_loss‚0.5451565171583008 0loss/FRt2/assignment_loss‚0.4480079542973803 /loss/FRt1/detection_loss‚0.4802155157437662 0loss/FRt2/detection_loss‚0.48920867716476196 % _timestamp‚1.7866627076850843e+09 &loss/total_loss‚1.9625886643642092 epoch‚38 trainer/global_step‚15249 _runtime‚4056.215010192ÒC){‚$õ 0loss/FRt1/assignment_loss‚0.5451565171583008 0loss/FRt2/assignment_loss‚0.4480079542973803 /loss/FRt1/detection_loss‚0.4802155157437662 0loss/FRt2/detection_loss‚0.48920867716476196 % _timestamp‚1.7866627076850843e+09 &loss/total_loss‚1.9625886643642092 epoch‚38 trainer/global_step‚15249 _runtime‚4056.215010192 _step‚1257é (gšÊ•ƒ$‚(:Š ¼žùÓôù•á gpu.0.gpu‚23.0 gpu.0.memory‚5, gpu.0.memoryAllocated‚7.0313262939453125) gpu.0.memoryAllocatedBytes‚ 6039863296 gpu.0.temp‚34.0 gpu.0.powerWatts‚119.176' gpu.0.enforcedPowerLimitWatts‚500.0 gpu.0.powerPercent‚23.8352 gpu.0.smClock‚1410 gpu.0.memoryClock‚1593$ gpu.0.correctedMemoryErrors‚1955# gpu.0.uncorrectedMemoryErrors‚0nºæ[“„$‚(:ˆ ¼žùÓÝÕ»ã# proc.memory.rssMB‚ 2315.17578125 proc.cpu.threads‚17 network.sent‚ 1.9883332e+07( disk./.usagePercent‚32.6896652810037 disk.sda2.in‚0! disk.dm-0.out‚60769.833984375$ disk.nvme0c0n1.in‚44033.58203125$ memory_percent‚68.35331916809082* proc.memory.percent‚14.130711555480957 network.recv‚ 7.642305e+06% disk./.usageGB‚487.09963607788086 disk.sda2.out‚0$ disk.nvme0n1.out‚60769.833984375 disk.drbd2165.in‚0 disk.drbd2165.out‚0 disk.dm-0.in‚44033.58203125 disk.sda.in‚ 37269.1484375 disk.sda1.out‚0" disk.nvme0n1.in‚44033.58203125 disk.dm-1.in‚ 37269.1484375 disk.dm-1.out‚ 74.578125 disk.sda3.in‚ 37269.1484375 disk.sda3.out‚ 74.578125 cpu‚27.232911809011227 disk.sda1.in‚0& disk.nvme0c0n1.out‚60769.833984375 disk.sda.out‚ 74.578125( proc.memory.availableMB‚ 5184.9921875ø- 2…$- _wandbruntime‚4068  _runtime‚4068¿L•„Ɇ$à & lr-AdamW/pg1‚0.0002030267162043611 & lr-AdamW/pg2‚0.0002030267162043611 trainer/global_step‚15299 % _timestamp‚1.7866627196243577e+09 _runtime‚4068.154350845 _step‚1258"Àëó·$È & lr-AdamW/pg1‚0.0002030267162043611 & lr-AdamW/pg2‚0.0002030267162043611 trainer/global_step‚15299 % _timestamp‚1.7866627196243577e+09 _runtime‚4068.154350845 _step‚1258ê zõar2ˆ$- _wandbruntime‚4068  _runtime‚4068ú7’•Å'‰$¿O XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚12839173 PCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚58387380 M@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚59049984 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4289 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚20643840 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚322 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚223 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚364011009507328 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚51366628 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚48119773 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚421298232229376 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 845178880 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 4628521206 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 5494538240 G>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4876 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚35280671 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚35280669 BDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚415126912139264 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 125829120 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 125829120 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 125829120 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚4287 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚417597962747904 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚61478912 trainer/global_step‚15299 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚322 @7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚5264 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚56666149 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚417597979852800 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚46135296 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚ 108028488 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚417597979852800 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17104896 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚429 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 5494538240 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 D;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚5264 IBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚60 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 5494538240 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 _runtime‚4068.158863283 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚51366628 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚56666147 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚367924040867840 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚262 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚421298232229376 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 3700311426560 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚ 108032777 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚4287 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 5368709120 _step‚1259 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚48119844 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚41945088 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚41945088ÃÔ­X¹ Š$ÄO KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 IBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚71 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚51362341 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 4552982528 PIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚69 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 122905600 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 5368709120 % _timestamp‚1.7866627196252334e+09 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 3700269481472 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚415126929178624 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚262 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚51362341 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚262 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 122905600 JDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚415126912139264 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 125829120 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 125829120 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 125829120 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚4287 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚417597962747904 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚61478912 trainer/global_step‚15299 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚322 @7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚5264 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚56666149 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚417597979852800 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚46135296 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚ 108028488 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚417597979852800 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17104896 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚429 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 5494538240 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 D;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚5264 IBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚60 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 5494538240 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 _runtime‚4068.158863283 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚51366628 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚56666147 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚367924040867840 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚262 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚421298232229376 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 3700311426560 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚ 108032777 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚4287 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 5368709120 _step‚1259 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚48119844 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚41945088 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚41945088 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚12839173 PCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚58387380 M@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚59049984 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4289 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚20643840 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚322 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚223 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚364011009507328 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚51366628 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚48119773 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚421298232229376 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 845178880 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 4628521206 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 5494538240 G>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4876 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚35280671 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚35280669 BDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 5368709120 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚223 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚12839929 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚421324844688896 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 108039623 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521206 # _timestamp‚1.78666271978676e+09 _runtime‚ 4068.31709332 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚ 108035328 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚322 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚ 108039623 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚51365628 I@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚4876 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚48122803 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚429 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚209 DDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59053056 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 5494538240 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚41948160 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 E6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚322 P6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚322 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚262 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚364034036236288 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 122306526 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 D=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚60 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚4293 NADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚58387428 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 4552982528 SFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚60484608 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 125829120 trainer/global_step‚15299 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654602240 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 3700500550144 NGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚54 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚415153114972160 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚417624285085696 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚367947396901888 IBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚60 G@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚60 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚56669702 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 EDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 G@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚56 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 3913340021760 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚421324844688896 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 4552982528 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚322 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚56669702 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 845178880 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 _step‚1260 >5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5264 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 3700500550144 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚41348068 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 5368709120 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚51365628 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚417624285085696 B9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚5264 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚367947336417280 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 831258624 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 3692838893226 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚35282876 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0ñgƒZÑ&$ËM UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚41948160 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 E6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚322 P6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚322 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚262 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚364034036236288 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 122306526 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 D=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚60 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚4293 NADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚58387428 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 4552982528 SFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚60484608 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 125829120 trainer/global_step‚15299 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654602240 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 3700500550144 NGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚54 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚415153114972160 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚417624285085696 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚367947396901888 IBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚60 G@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚60 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚56669702 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 EDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 G@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚56 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 3913340021760 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚421324844688896 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 4552982528 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚322 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚56669702 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 845178880 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 _step‚1260 >5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5264 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 3700500550144 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚41348068 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 5368709120 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚51365628 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚417624285085696 B9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚5264 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚367947336417280 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 831258624 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 3692838893226 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚35282876 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 122905600 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚51369921 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚48122859 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚418845953865386 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚417624302190592 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚417624302190592 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚12839983 H@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚665 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 5368709120 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚223 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚12839929 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚421324844688896 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 108039623 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521206 # _timestamp‚1.78666271978676e+09 _runtime‚ 4068.31709332 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚ 108035328 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚322 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚ 108039623 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚51365628 I@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚4876 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚48122803 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚429 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚209 DDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59053056 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 5494538240ì ’Sç2Ž$- _wandbruntime‚4068  _runtime‚4068ÈCu$ï /loss/FRt1/detection_loss‚0.5059787239346537 /loss/FRt2/detection_loss‚0.4883710994473746 %loss/total_loss‚1.789593834792324 epoch‚38 _step‚1261 1loss/FRt1/assignment_loss‚0.41282336901674943 trainer/global_step‚15299 $ _timestamp‚1.786662719787826e+09 _runtime‚4068.317646475 1loss/FRt2/assignment_loss‚0.38242064239354623£<ª¼z$ô 1loss/FRt1/assignment_loss‚0.41282336901674943 trainer/global_step‚15299 $ _timestamp‚1.786662719787826e+09 _runtime‚4068.317646475 1loss/FRt2/assignment_loss‚0.38242064239354623 /loss/FRt1/detection_loss‚0.5059787239346537 /loss/FRt2/detection_loss‚0.4883710994473746 %loss/total_loss‚1.789593834792324 epoch‚38 _step‚1261í ß𧖑$‚(:‹ ËžùÓ¡§ëà gpu.0.gpu‚60.0 gpu.0.memory‚11, gpu.0.memoryAllocated‚7.0313262939453125) gpu.0.memoryAllocatedBytes‚ 6039863296 gpu.0.temp‚35.0 gpu.0.powerWatts‚202.446' gpu.0.enforcedPowerLimitWatts‚500.0 gpu.0.powerPercent‚40.4892 gpu.0.smClock‚1410 gpu.0.memoryClock‚1593$ gpu.0.correctedMemoryErrors‚1955# gpu.0.uncorrectedMemoryErrors‚0Pÿ›’$‚(: ËžùÓÁÁ¬ã disk.dm-1.out‚ 74.85546875 disk.sda3.out‚ 74.85546875 disk.sda.in‚38394.64453125 disk.sda.out‚ 74.85546875 disk.dm-0.in‚ 44052.5078125! disk.nvme0n1.in‚ 44052.5078125$ memory_percent‚68.35355758666992' proc.memory.availableMB‚ 5184.953125* proc.memory.percent‚14.130711555480957 cpu‚27.232672291749235 proc.cpu.threads‚17 disk.drbd2165.in‚0 disk.drbd2165.out‚0' disk.nvme0c0n1.out‚60773.4638671875 disk.sda1.in‚0 disk.sda1.out‚0 disk.sda2.in‚0 disk.sda2.out‚0% disk.nvme0n1.out‚60773.4638671875 network.sent‚ 1.9952598e+07" disk.dm-0.out‚60773.4638671875 disk.sda3.in‚38394.64453125# disk.nvme0c0n1.in‚ 44052.5078125# proc.memory.rssMB‚ 2315.17578125 network.recv‚ 7.668494e+06( disk./.usagePercent‚32.6887060207729% disk./.usageGB‚487.08534240722656 disk.dm-1.in‚38394.64453125ÓLìß2“$- _wandbruntime‚4080  _runtime‚4080ŸŠ? Ë”$Å ' lr-AdamW/pg2‚0.00019893182571640192 trainer/global_step‚15349 % _timestamp‚1.7866627319704633e+09 _runtime‚4080.500466367 _step‚1262 ' lr-AdamW/pg1‚0.00019893182571640192†¥›Ð•$Ê trainer/global_step‚15349 % _timestamp‚1.7866627319704633e+09 _runtime‚4080.500466367 _step‚1262 ' lr-AdamW/pg1‚0.00019893182571640192 ' lr-AdamW/pg2‚0.00019893182571640192î OÕG2–$- _wandbruntime‚4080  _runtime‚4080ÛcÆ‹m —$¿O L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚51527443 G>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4876 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17104896 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚ 108375529 BDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 3704128783154 KDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚60 PCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚59049984 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚365163357503488 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚262 _runtime‚4080.504891969 JBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚665 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 PIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚65 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚48270689 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚35390919 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 3704170131174 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 5368709120 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚422625994934784 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 122905600 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 122306526 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚48270756 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 3925344060416 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 3711898192384 JDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 M@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚59049984 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 4654602240 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 125829120 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚322 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚4287 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 4628521206 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 5494538240 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 5368709120 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚262 IBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚60 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 4654602240 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚418914079637504 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4289 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 125829120 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 5494538240 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚322 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚41945088 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 4552982528 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚56843797 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚ 108371240 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚41945088 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚416436201816064 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚262 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚44038144 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚56843797 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17104896 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚128O(¡X79770 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 4552982528 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚4287 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚429 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚12879835 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 3711856247296 D;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚5264 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 831258624 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 4517085184 @7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚5264 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚422625935884800 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚209 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚61478912 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚420140330599218 % _timestamp‚1.7866627319713738e+09 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚56843799 UHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚64681984 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 5368709120 C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚4289 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚51527443 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚41348020 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 trainer/global_step‚15349¦+U^Ê'˜$ÄO \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚44038144 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚56843797 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17104896 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚12879770 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 4552982528 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚4287 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚429 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚12879835 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 3711856247296 D;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚5264 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 831258624 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 4517085184 @7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚5264 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚422625935884800 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚209 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚61478912 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚420140330599218 % _timestamp‚1.7866627319713738e+09 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚56843799 UHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚64681984 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 5368709120 C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚4289 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚51527443 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚41348020 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 trainer/global_step‚15349 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚51527443 G>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4876 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17104896 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚ 108375529 BDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 3704128783154 KDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚60 PCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚59049984 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚365163357503488 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚262 _runtime‚4080.504891969 JBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚665 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 PIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚65 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚48270689 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚35390919 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 3704170131174 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 5368709120 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚422625994934784 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 122905600 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 122306526 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚48270756 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 3925344060416 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 3711898192384 JDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 M@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚59049984 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 4654602240 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 125829120 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚322 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚4287 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 4628521206 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 5494538240 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 5368709120 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚262 IBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚60 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 4654602240 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚418914079637504 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4289 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 125829120 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 5494538240 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚322 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚41945088 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 4552982528 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚56843797 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚ 108371240 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚41945088 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚416436201816064 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚262ï ªyÊ»2™$- _wandbruntime‚4080  _runtime‚4080f\¼FÎ&š$ÈM REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚35393124 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 5368709120 _step‚1264 G@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚71 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚209 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 BDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654602240 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚418940401975296 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 108382375 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 845178880 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚420166746965366 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 122306526 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚223 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚369111990985728 DDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59053056 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 3712129264128 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 3712087315968 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚422652548344320 NADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚59053056 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 125829120 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚51535023 I@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚4876 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚51530730 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17104896 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚12880666 trainer/global_step‚15349 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚56847352 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚4293 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚35393126 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚420166805352794 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 3704400703834 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚56847352 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 5494538240 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 5368709120 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 P6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚322 EDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521206 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 4654602240 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚418940419080192 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 3712129264128 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚429 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚20643840 _runtime‚4080.664109592 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 3925586109440 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚ 108378080 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚4295 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚56847350 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚418940401975296 >5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5264 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚56847350 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚262 H@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚665 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 5494538240 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚48273792 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚365186425520128 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 3925632241664 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚46132224 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚61478912 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚41948160 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 122905600 NGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚69 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 125829120 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚322 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚4293 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚48273721 NADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚58387428 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17104896 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚51530730 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 4517085184 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚418940419080192 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚ 108378080 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚51535023 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚41348068 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 3704359355766 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚262 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚416462404648960 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚422652489291264 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 3712087315968 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚369112057761792 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚422652548344320 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚322 $ _timestamp‚1.786662732133779e+09;<Þ°Ó&›$ÍM DDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59053056 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 3712129264128 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 3712087315968 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚422652548344320 NADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚59053056 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 125829120 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚51535023 I@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚4876 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚51530730 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17104896 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚12880666 trainer/global_step‚15349 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚56847352 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚4293 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚35393126 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚420166805352794 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 3704400703834 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚56847352 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 5494538240 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 5368709120 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 P6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚322 EDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521206 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 4654602240 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚418940419080192 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 3712129264128 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚429 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚20643840 _runtime‚4080.664109592 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 3925586109440 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚ 108378080 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚4295 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚56847350 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚418940401975296 >5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5264 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚56847350 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚262 H@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚665 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 5494538240 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚48273792 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚365186425520128 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 3925632241664 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚46132224 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚61478912 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚41948160 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 122905600 NGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚69 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 125829120 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚322 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚4293 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚48273721 NADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚58387428 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17104896 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚51530730 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 4517085184 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚418940419080192 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚ 108378080 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚51535023 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚41348068 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 3704359355766 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚262 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚416462404648960 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚422652489291264 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 3712087315968 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚369112057761792 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚422652548344320 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚322 $ _timestamp‚1.786662732133779e+09 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚35393124 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 5368709120 _step‚1264 G@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚71 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚209 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 BDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654602240 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚418940401975296 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 108382375 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 845178880 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚420166746965366 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 122306526 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚223 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚369111990985728ð 6àad2œ$- _wandbruntime‚4080  _runtime‚4080jGjv$ð 0loss/FRt2/assignment_loss‚0.4094290557634085 epoch‚38 trainer/global_step‚15349 0loss/FRt1/assignment_loss‚0.5027772020964212 /loss/FRt1/detection_loss‚0.5327292699439978 0loss/FRt2/detection_loss‚0.48195059109594884 &loss/total_loss‚1.9268861188997763 % _timestamp‚1.7866627321420937e+09 _runtime‚4080.671982082 _step‚1265§5+ž$õ /loss/FRt1/detection_loss‚0.5327292699439978 0loss/FRt2/detection_loss‚0.48195059109594884 &loss/total_loss‚1.9268861188997763 % _timestamp‚1.7866627321420937e+09 _runtime‚4080.671982082 _step‚1265 0loss/FRt2/assignment_loss‚0.4094290557634085 epoch‚38 trýMPØPainer/global_step‚15349 0loss/FRt1/assignment_loss‚0.5027772020964212ñ =2ùŒ2Ÿ$- _wandbruntime‚4092  _runtime‚4092¯¦:Ë $Å ' lr-AdamW/pg1‚0.00019487232930752194 ' lr-AdamW/pg2‚0.00019487232930752194 trainer/global_step‚15399 % _timestamp‚1.7866627441019716e+09 _runtime‚4092.632022428 _step‚12661suÚС$Ê % _timestamp‚1.7866627441019716e+09 _runtime‚4092.632022428 _step‚1266 ' lr-AdamW/pg1‚0.00019487232930752194 ' lr-AdamW/pg2‚0.00019487232930752194 trainer/global_step‚15399ò y¤€2¢$- _wandbruntime‚4092  _runtime‚4092&™üyÅ'£$¿O GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚423953639540224 KBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚4876 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 5494538240 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚57021447 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚35501171 KDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚60 IBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚60 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚423953639540224 G>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4876 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚12920354 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚41945088 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 3715731941782 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚ 108713992 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 831258624 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 3937661509632 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚421461182086626 F>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚12920420 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚322 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚262 C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚4289 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚20643840 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚61478912 _runtime‚4092.636899034 PIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚66 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 5494538240 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚429 trainer/global_step‚15399 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚57021449 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚423953698590208 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 122905600 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 3715690593762 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 5368709120 IBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚68 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 PCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚58387380 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 5494538240 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 5368709120 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 5368709120 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 125829120 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚420230213632000 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚57021447 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 845178880 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 4517085184 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚262 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 3723443013120 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 122905600 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚423953698590208 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚51692545 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚420230196527104 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚322 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚417745491492864 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 4654602240 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚ 108718281 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4289 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 % _timestamp‚1.7866627441029437e+09 _step‚1267 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚51696832 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚51696832 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 PCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚59049984 @7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚5264 D;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚5264 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚46135296 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚41348020 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 122306526 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 4552982528 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚41945088 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚417745508532224 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 4552982528 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 UHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚66779136 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 3937615374336 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚420230213632000 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17104896 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚4287 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚370253341517824 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚322 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚223 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚421461240474006 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 125829120 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 3723484958208 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚51692545 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚366315746787328 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 125829120 F?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚60 M@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚59049984 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚370253408296960 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 BDeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4289 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 % _timestamp‚1.7866627441029437e+09 _step‚1267 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚51696832 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚51696832 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 PCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚59049984 @7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚5264 D;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚5264 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚46135296 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚41348020 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 122306526 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 4552982528 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚41945088 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚417745508532224 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 4552982528 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 UHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚66779136 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 3937615374336 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚420230213632000 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17104896 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚4287 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚370253341517824 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚322 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚223 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚421461240474006 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 125829120 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 3723484958208 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚51692545 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚366315746787328 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 125829120 F?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚60 M@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚59049984 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚370253408296960 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 BDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚423953639540224 KBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚4876 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 5494538240 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚57021447 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚35501171 KDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚60 IBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚60 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚423953639540224 G>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4876 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚12920354 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚41945088 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 3715731941782 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚ 108713992 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 831258624 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 3937661509632 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚421461182086626 F>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚12920420 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚322 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚262 C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚4289 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚20643840 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚61478912 _runtime‚4092.636899034 PIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚66 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 5494538240 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚429 trainer/global_step‚15399 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚57021449 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚423953698590208 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 122905600 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 3715690593762 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 5368709120 IBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚68 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 PCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚58387380 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 5494538240 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 5368709120 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 5368709120 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 125829120 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚420230213632000 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚57021447 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 845178880 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 4517085184 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚262 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 3723443013120 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 122905600 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚423953698590208ó ‚enš2¥$- _wandbruntime‚4092  _runtime‚4092ñ¶ãÿÏ&¦$ÉM H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚ 108725127 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚423980251999744 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚370276696339456 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 3715962514442 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 125829120 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚12921235 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 125829120 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚262 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚322 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 3723674081792 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 3723716029952 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚57025000 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 845178880 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚35646464 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 3937866532864 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 122306526 EDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654602240 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚57025000 EDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 % _timestamp‚1.7866627442629197e+09 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 5368709120 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚4295 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚366338794160128 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚223 _step‚1268 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚ 108720832 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚51700125 _runtime‚4092.793167783 >5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5264 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 122905600 D=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚60 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚370276640049152 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚48424611 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚262 trainer/global_step‚15399 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚35503376 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 5494538240 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17104896 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚51695832 G@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚62 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 125829120 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚ 108720832 BDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 108725127 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚366338773516288 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚61478912 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚35503374 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521206 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚322 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 I@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚4876 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 3723716029952 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚322 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚417771677286400 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚51695832 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚209 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚417771694325760 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 4517085184 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚420256535969792 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 122905600 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17104896 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 5368709120 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 PDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59053056 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚429 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚12921175 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 3723674081792 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚20643840 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚322 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚421487598452774 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 4654602240 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚420256535969792 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 4552982528 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚41948160 SFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚56290304 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 3937902179328 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚41948160 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 5368709120 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 4552982528 B9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚5264 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚48424549 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚421487656840202 IBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚60KðÍmÛ§$ÎM NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚57025002 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 K>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59053056 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚429 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚12921175 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 3723674081792 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚20643840 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚322 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚421487598452774 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 4654602240 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚420256535969792 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 4552982528 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚41948160 SFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚56290304 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 3937902179328 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚41948160 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 5368709120 QBDeviceStatsMonitor.on_train_batch_end/active_bytkªÙáùes.large_pool.peak‚ 4552982528 B9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚5264 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚48424549 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚421487656840202 IBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚60 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚ 108725127 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚423980251999744 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚370276696339456 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 3715962514442 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 125829120 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚12921235 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 125829120 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚262 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚322 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 3723674081792 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 3723716029952 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚57025000 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 845178880 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚35646464 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 3937866532864 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 122306526 EDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654602240 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚57025000 EDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 % _timestamp‚1.7866627442629197e+09 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 5368709120 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚4295 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚366338794160128 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚223 _step‚1268 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚ 108720832 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚51700125 _runtime‚4092.793167783 >5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5264 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 122905600 D=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚60 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚370276640049152 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚48424611 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚262 trainer/global_step‚15399 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚35503376 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 5494538240 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17104896 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚51695832 G@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚62 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 125829120 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚ 108720832 BDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 108725127 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚366338773516288 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚61478912 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚35503374 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521206 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚322 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 I@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚4876 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 3723716029952 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚322 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚417771677286400 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚51695832 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚209 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚417771694325760 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 4517085184 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚420256535969792 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 122905600 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17104896 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 5368709120 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 PDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4876 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 3735071724032 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚425281343195648 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚421546313416704 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚367468115427328 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚322 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 3735029778944 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 5494538240 KDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚60 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚57199099 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚4287 PIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚61 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 845178880 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚367468094783488 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 3727252404370 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚223 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 3949963171840 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚61478912 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 125829120 JDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚425281343195648 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚421546330521600 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 5368709120 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 5368709120 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚262 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 4552982528 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚51861934 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 @7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚5264 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 4654602240 JBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚665 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚48572789 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚419054781169664 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 125829120 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚41945088 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 4552982528 PCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚58387380 _runtime‚4105.720888973 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚51857647 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 122905600 UHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚58390528 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚422782033574034 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17104896 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚12961368 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚371418078599168 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚20643840 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚425281402245632 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚ 109061033 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚51861934 IBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚63 trainer/global_step‚15449 JDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚57199099 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 KBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚4876 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚35611421 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 125829120 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚425281402245632 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚421546313416704 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 3735029778944 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚12961307 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 3949925425152 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚41348020 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚322 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17104896 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4289 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚262 M@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚59049984 F>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 D;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚5264 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚48572726åò "Ê'²$ÄO ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 831258624 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 _step‚1271 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚425281343195648 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚421546330521600 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 5368709120 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 5368709120 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚262 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 4552982528 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚51861934 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 @7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚5264 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 4654602240 JBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚665 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚48572789 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚419054781169664 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 125829120 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚41945088 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 4552982528 PCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚58387380 _runtime‚4105.720888973 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚51857647 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 122905600 UHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚58390528 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚422782033574034 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17104896 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚12961368 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚371418078599168 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚20643840 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚425281402245632 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚ 109061033 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚51861934 IBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚63 trainer/global_step‚15449 JDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚57199099 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 KBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚4876 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚35611421 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 125829120 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚425281402245632 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚421546313416704 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 3735029778944 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚12961307 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 3949925425152 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚41348020 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚322 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17104896 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4289 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚262 M@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚59049984 F>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 D;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚5264 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚48572726 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚429 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 5368709120 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 4628521206 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 122306526 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚262 F?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚60 % _timestamp‚1.7866627571868944e+09 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚421546330521600 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚209 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚422782091961414 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 4517085184 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 5494538240 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚322 G>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4876 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 3735071724032 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚425281343195648 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚421546313416704 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚367468115427328 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚322 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 3735029778944 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 5494538240 KDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚60 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚57199099 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚4287 PIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚61 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 845178880 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚367468094783488 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 3727252404370 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚223 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 3949963171840 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚61478912 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 125829120 JDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521206 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚419080966963200 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚425307955655168 >5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5264 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 3735302795776 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚209 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 EDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654602240 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚367491142156288 NADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚59053056 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 IBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚60 K>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59053056 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 122905600 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚61478912 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 5368709120 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚322 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚4293 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 109067879 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚322 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚ 109067879 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚51860934 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚48575787 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚429 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 D=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚60 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚51860934 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 4654602240 NGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚70 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚322 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚422808508327610 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚419080984002560 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚57202652 B9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚5264 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 _runtime‚4105.878868081 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚421572635754496 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚41948160 H@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚665 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚367491162800128 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 3727482976982 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 5494538240 DDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚322 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚262 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚425307896602112 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚421572652859392 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚12962163 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚41937920 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 125829120 $ _timestamp‚1.786662757348527e+09 ÖDÓ&µ$ÍM J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚57202650 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 3735260847616 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚371441312415744 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚20643840 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚422808449940182 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17104896 EDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521206 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚419080966963200 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚425307955655168 >5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5264 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 3735302795776 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚209 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 EDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654602240 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚367491142156288 NADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚59053056 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 IBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚60 K>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59053056 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 122905600 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚61478912 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 5368709120 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚322 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚4293 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 109067879 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚322 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚ 109067879 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚51860934 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚48575787 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚429 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 D=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚60 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚51860934 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 4654602240 NGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚70 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚322 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚422808508327610 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚419080984002560 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚57202652 B9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚5264 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 _runtime‚4105.878868081 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚421572635754496 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚41948160 H@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚665 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚367491162800128 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 3727482976982 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 5494538240 DDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚322 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚262 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚425307896602112 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚421572652859392 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚12962163 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚41937920 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 125829120 $ _timestamp‚1.786662757348527e+09 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚421572635754496 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 3735302795776 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚51865227 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚41348068 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 5494538240 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 125829120 G@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚60 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚371441374997504 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 845178880 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 3950212197376 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚262 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 831258624 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 trainer/global_step‚15449 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚4295 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚425307955655168 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚35613626 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 3950170259456 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2ø íÂhJ2¶$- _wandbruntime‚4105  _runtime‚4105s±Ú³w·$ñ 0loss/FRt1/detection_loss‚0.49251813334135086 %loss/total_loss‚1.868636899284792 epoch‚38 trainer/global_step‚15449 % _timestamp‚1.7866627573494678e+09 1loss/FRt2/assignment_loss‚0.41598589371287814 /loss/FRt2/detection_loss‚0.4693065111753727 _runtime‚4105.879360121 _step‚1273 1loss/FRt1/assignment_loss‚0.49082636105519023&¶&/|¸$ö 0loss/FRt1/detection_loss‚0.49251813334135086 %loss/total_loss‚1.868636899284792 epoch‚38 trainer/global_step‚15449 % _timestamp‚1.7866627573494678e+09 1loss/FRt2/assignment_loss‚0.41598589371287814 /loss/FRt2/detection_loss‚0.4693065111753727 _runtime‚4105.879360121 _step‚1273 1loss/FRt1/assignment_loss‚0.49082636105519023ù ý\µ¨¡¹$‚(:– éžùÓöȺá gpu.0.gpu‚62.0 gpu.0.memory‚11, gpu.0.memoryAllocated‚7.0313262939453125) gpu.0.memoryAllocatedBytes‚ 6039863296 gpu.0.temp‚35.0 gpu.0.powerWatts‚233.782' gpu.0.enforcedPowerLimitWatts‚500.0) gpu.0.powerPercent‚46.756400000000006 gpu.0.smClock‚1410 gpu.0.memoryClock‚1593$ gpu.0.correctedMemoryErrors‚1955# gpu.0.uncorrectedMemoryErrors‚0€‘”º$‚(:‰ éžùÓ͵»å disk.sda1.in‚0! proc.memory.rssMB‚ 2253.046875 proc.cpu.threads‚17* disk./.usagePercent‚32.694066306524824$ disk./.usageGB‚487.1652145385742 disk.nvme0c0n1.in‚ 44055.9375 disk.sda.out‚ 75.60546875 disk.sda1.out‚0 disk.dm-1.out‚ 75.60546875 disk.drbd2165.out‚0 cpu‚27.232165807860884 network.recv‚ 7.728278e+06# disk.dm-0.out‚60897.62939453125 disk.sda3.out‚ 75.60546875 disk.sda2.out‚0 disk.nvme0n1.in‚ 44055.9375& disk.nvme0n1.out‚60897.62939453125 disk.dm-1.in‚38394.64453125 disk.drbd2165.in‚0 network.sent‚ 2.0118504e+07( disk.nvme0c0n1.out‚60897.62939453125 disk.sda.in‚38394.64453125 disk.sda2.in‚0$ memory_percent‚67.97285079956055' proc.memory.availableMB‚ 5247.328125* proc.memory.percent‚13.751506805419922 disk.dm-0.in‚ 44055.9375 disk.sda3.in‚38394.64453125 þz2»$- _wandbruntime‚4118  _runtime‚4118lö˼$Å ' lr-AdamW/pg1‚0.00018686055937243244 ' lr-AdamW/pg2‚0.00018686055937243244 trainer/global_step‚15499 % _timestamp‚1.7866627694865222e+09 _runtime‚4118.016515407 _step‚12740Ùø×н$Ê % _timestamp‚1.7866627694865222e+09 _runtime‚4118.016515407 _step‚1274 ' lr-AdamW/pg1‚0.00018686055937243244 ' lr-AdamW/pg2‚0.00018686055937243244 trainer/global_step‚15499ú DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 PIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚75 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 831258624 JDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚52027036 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚422862447411200 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 4552982528 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚61478912 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚41945088 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚372582772443648 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚426609046851072 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚37746688 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚426609105901056 M@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚59049984 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚209 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚322 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚322 F?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚60 KBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚4876 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚48723969 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 3738855562998 KDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚60 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚57376747 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 3746658489856 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 122905600 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚57376749 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 4654602240 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚422862430306304 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4289 BDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4876 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 3746616544768 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚368620463423488 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 3962250629632 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚4287'¬Ä•yÀ$ÃO PCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚59049984 UHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚58390528 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 125829120 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 4654602240 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚52022749 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 3746616544768 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚13002223 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 845178880 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚20643840 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚322 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚262 _runtime‚ 4118.02087499 F>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 PIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚75 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 83125^þ²P"8624 JDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚52027036 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚422862447411200 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 4552982528 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚61478912 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚41945088 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚372582772443648 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚426609046851072 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚37746688 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚426609105901056 M@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚59049984 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚209 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚322 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚322 F?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚60 KBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚4876 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚48723969 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 3738855562998 KDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚60 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚57376747 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 3746658489856 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 122905600 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚57376749 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 4654602240 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚422862430306304 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4289 BDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4876 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 3746616544768 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚368620463423488 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 3962250629632 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚4287 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 3962288376320 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 122306526û (Òþc2Á$- _wandbruntime‚4118  _runtime‚4118:l_Ï&Â$ÉM NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚52026036 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521206 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 4552982528 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 831258624 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚61478912 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 3962541596160 DDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 125829120 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚4293 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 4654602240 I@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚4876 NGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚71 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚426635600257536 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚41948160 H@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚665 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚35723876 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 5494538240 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚57380302 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 3746847613440 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚368643510796288 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚424129359815018 G@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚73 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚420390273679360 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚ 109410631 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 122905600 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚13003028 G@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚60 EDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59053056 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚422888769748992 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 3962499658240 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚322 BDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 3746847613440 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚13003099 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 122306526 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚262 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚426635659310592 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚422888752644096 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 845178880 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚ 109406336 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚368643531440128 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 125829120 % _timestamp‚1.7866627696519485e+09 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚48726902 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 109410631 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚420390256640000 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 5368709120 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚322 >5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5264 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 _runtime‚4118.182362744 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚4293 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚209 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚426635659310592 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚372606010454528 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚322 D=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚60 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚422888769748992 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚20643840 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654602240 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 3746889561600 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚262 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚422888752644096 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 5368709120 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 3746889561600 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 122905600 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚52030329cØ|Ô&Ã$ÎM G@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚73 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚420390273679360 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚ 109410631 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 122905600 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚13003028 G@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚60 EDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59053056 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚422888769748992 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 3962499658240 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚322 BDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 3746847613440 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚13003099 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 122306526 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚262 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚426635659310592 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚422888752644096 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 845178880 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚ 109406336 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚368643531440128 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 125829120 % _timestamp‚1.7866627696519485e+09 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚48726902 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 109410631 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚420390256640000 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 5368709120 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚322 >5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5264 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 _runtime‚4118.182362744 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚4293 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚209 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚426635659310592 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚372606010454528 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚322 D=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚60 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚422888769748992 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚20643840 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654602240 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 3746889561600 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚262 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚422888752644096 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 5368709120 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 3746889561600 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 122905600 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚52030329 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚52026036 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521206 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 4552982528 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 831258624 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚61478912 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 3962541596160 DDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 125829120 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚4293 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 4654602240 I@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚4876 NGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚71 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚426635600257536 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚41948160 H@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚665 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚35723876 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 5494538240 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚57380302 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 3746847613440 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚368643510796288 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚424129359815018ü ´KU¼2Ä$- _wandbruntime‚4118  _runtime‚4118ÿ`œDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4876 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚41945088 KBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚4876 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚223 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 IBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚60 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚41945088 JBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚665 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 4654602240 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 3750417373606 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 3750376025586 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 125829120 JDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚209 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 5368709120 C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚4289 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚ 109742248 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 4552982528 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚425423794936230 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 5368709120 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚262 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚424178564300800 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 3758245255680 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚429 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 4517085184 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚13043149 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17104896 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚424178547195904 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 122905600 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚48875068 PIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚71 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚427936750506496 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 3758203310592 D;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚5264 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚57554397 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 4654602240 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17104896 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚50329600 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚425423736548850 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚427936809556480 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4289 IBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚73 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 3974577423872 BDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4876 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚41945088 KBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚4876 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚223 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 IBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚60 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚41945088 JBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚665 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 4654602240 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 3750417373606 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 3750376025586 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 125829120 JDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚209 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 5368709120 C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚4289 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚ 109742248 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 4552982528 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚425423794936230 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 5368709120 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚262 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚424178564300800 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 3758245255680 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚429 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 4517085184 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚13043149 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17104896 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚424178547195904 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 122905600 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚48875068 PIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚71 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚427936750506496 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 3758203310592 D;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚5264 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚57554397 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 4654602240 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17104896 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚50329600 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚425423736548850 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚427936809556480 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4289 IBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚73 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 3974577423872 BDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 109753383 H@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚665 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 5368709120 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚424204886638592 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚424204869533696 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 5368709120 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚52191138 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17104896 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 4552982528 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚41348068 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚52195431 G@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚75 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚35834124 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚209 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 3750606598198 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚262 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚52195431 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 D=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚60 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 3758476327424 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 122905600 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 3974823655936 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚322 IBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚60 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚ 109753383 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚425450152914998 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚52191138 >5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5264 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚57557950 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 4552982528 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚427963303912960 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 125829120 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚57557952 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 4654602240 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚369795879436288 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚322 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚41948160 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚41948160 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚ 109749088 B9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚5264 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚373770723736064 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚373770654862848 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 trainer/global_step‚15549 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚20643840 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚322 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚262 _runtime‚4130.487094353 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚369795900080128 BDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 _step‚1280 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚ 109749088 EDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59053056 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚57557950 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 831258624 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚61478912 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 5368709120 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚4293 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚48878089 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚425450211302426 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 122306526 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 5494538240 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚424204869533696 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654602240 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚223 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚262 PDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521206 NADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚59053056 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 845178880 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚421699563356160 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 125829120 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚424204886638592 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚13044038 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚322 % _timestamp‚1.7866627819567053e+09 D8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17104896 SFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚68873216 NADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚58387428 G@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚60 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 122905600 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚429 NGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚73 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 3974775426560 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 5494538240@"hÀzÑ$ÎM UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 D=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚60 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 3758476327424 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 122905600 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 3974823655936 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚322 IBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚60 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚ 109753383 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚425450152914998 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚52191138 >5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5264 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚57557950 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 4552982528 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚427963303912960 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 125829120 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚57557952 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 4654602240 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚369795879436288 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚322 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚41948160 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚41948160 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚ 109749088 B9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚5264 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚373770723736064 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚373770654862848 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 trainer/global_step‚15549 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚20643840 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚322 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚262 _runtime‚4130.487094353 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚369795900080128 BDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 _step‚1280 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚ 109749088 EDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59053056 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚57557950 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 831258624 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚61478912 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 5368709120 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚4293 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚48878089 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚425450211302426 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 122306526 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 5494538240 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚424204869533696 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654602240 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚223 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚262 PDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521206 NADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚59053056 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 845178880 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚421699563356160 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 125829120 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚424204886638592 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚13044038 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚322 % _timestamp‚1.7866627819567053e+09 D8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17104896 SFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚68873216 NADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚58387428 G@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚60 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 122905600 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚429 NGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚73 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 3974775426560 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 5494538240 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 109753383 H@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚665 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 5368709120 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚424204886638592 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚424204869533696 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 5368709120 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚52191138 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17104896 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 4552982528 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚41348068 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚52195431 G@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚75 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚35834124 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚209 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 3750606598198 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚262 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚52195431€ ^D*ô2Ò$- _wandbruntime‚4130  _runtime‚4130†š$tÓ$î 0loss/FRt2/assignment_loss‚0.4107052283336211 /loss/FRt1/detection_loss‚0.5227565410860894 /loss/FRt2/detection_loss‚0.4875452198259494 %loss/total_loss‚1.978366476497848 epoch‚38 trainer/global_step‚15549 % _timestamp‚1.7866627819581892e+09 0loss/FRt1/assignment_loss‚0.5573594872521881 _runtime‚4130.487982151 _step‚1281¹ØÂÃyÔ$ó _step‚1281 0loss/FRt2/assignment_loss‚0.4107052283336211 /loss/FRt1/detection_loss‚0.5227565410860894 /loss/FRt2/detection_loss‚0.4875452198259494 %loss/total_loss‚1.978366476497848 epoch‚38 trainer/global_step‚15549 % _timestamp‚1.7866627819581892e+09 0loss/FRt1/assignment_loss‚0.5573594872521881 _runtime‚4130.487982151 )š„1–Õ$‚(:‹ ‡ŸùÓƵ²á gpu.0.gpu‚63.0 gpu.0.memory‚11, gpu.0.memoryAllocated‚7.0313262939453125) gpu.0.memoryAllocatedBytes‚ 6039863296 gpu.0.temp‚35.0 gpu.0.powerWatts‚190.724' gpu.0.enforcedPowerLimitWatts‚500.0 gpu.0.powerPercent‚38.1448 gpu.0.smClock‚1410 gpu.0.memoryClock‚1593$ gpu.0.correctedMemoryErrors‚1955# gpu.0.uncorrectedMemoryErrors‚0Uoª”›Ö$‚(: ‡ŸùÓÕ¿³æ disk.drbd2165.out‚0 disk.sda2.out‚0* proc.memory.percent‚13.751506805419922 proc.cpu.threads‚17 network.sent‚ 2.0266261e+07 disk.sda1.in‚0 disk.sda1.out‚0" disk.nvme0n1.in‚44065.12890625 disk.dm-1.out‚ 76.015625) proc.memory.availableMB‚ 5246.94140625 disk.drbd2165.in‚0 disk.dm-0.in‚44065.12890625$ disk.nvme0c0n1.in‚44065.12890625 disk.sda.in‚38394.64453125& disk.nvme0n1.out‚60909.26123046875$ memory_percent‚67.97521114349365! proc.memory.rssMB‚ 2253.046875 cpu‚27.231623600365378 network.recv‚ 7.790596e+06# disk.dm-0.out‚60909.26123046875 disk.sda3.in‚38394.64453125 disk.sda3.out‚ 76.015625( disk.nvme0c0n1.out‚60909.26123046875 disk.sda.out‚ 76.015625 disk.sda2.in‚0 disk.dm-1.in‚38394.64453125) disk./.usagePercent‚32.69368741537415$ disk./.usageGB‚487.1595687866211ýÚç)2×$- _wandbruntime‚4142  _runtime‚4142{t\ÉØ$à _step‚1282 & lr-AdamW/pg1‚0.0001789934646900887 & lr-AdamW/pg2‚0.0001789934646900887 trainer/global_step‚15599 % _timestamp‚1.7866627941691096e+09 _runtime‚4142.699235492ªð»ÿÎÙ$È trainer/global_step‚15599 % _timestamp‚1.7866627941691096e+09 _runtime‚4142.699235492 _step‚1282 & lr-AdamW/pg1‚0.0001789934646900887 & lr-AdamW/pg2‚0.0001789934646900887‚ .«W2Ú$- _wandbruntime‚4142  _runtime‚4142•öÜ®Å'Û$¿O @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚41348020 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 5494538240 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 4552982528 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚13084182 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚322 @7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚5264 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚57732047 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚425494664085504 D;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚5264 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚52352953 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 3761979184214 KDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚60 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 3769790076416 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚322 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚49026421 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚35942169 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚422982667239424 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 5368709120 IBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚60 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚223 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 125829120 JDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 4654602240 IBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚70 _runtime‚4142.703941004 PCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚59049984 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚20643840 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚61478912 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 122306526 UHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚68876288 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚422982650200064 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 5368709120 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚322 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17104896 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚41945088 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 5494538240 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚262 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚429264513211904 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚57732049 BDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚429264454161920 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚426744646423638 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 JBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚665 M@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚59049984 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚4287 G>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4876 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚374912037237248 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚425494664085504 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚429 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚209 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 845178880 PCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚58387380 _step‚1283 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚ 110085000 KBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚4876 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 831258624 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 3769790076416 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚57732047 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚426744588036258 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 4628521206 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 5494538240 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚4289 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚49026351 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚322 trainer/global_step‚15599 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚425494681190400 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 122905600 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 122905600 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚57732049 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚35942171 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 125829120 F?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚60 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4289 PIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚68 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17104896 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚41945088 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 5368709120 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 4654602240 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 4552982528 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 % _timestamp‚1.7866627941701531e+09 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚429264513211904 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚ 110089289 JDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 4654602240 IBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚70 _runtime‚4142.703941004 PCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚59049984 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚20643840 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚61478912 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 122306526 UHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚68876288 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚422982650200064 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 5368709120 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚322 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17104896 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚41945088 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 5494538240 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚262 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚429264513211904 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚57732049 BDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚429264454161920 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚426744646423638 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 JBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚665 M@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚59049984 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚4287 G>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4876 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚374912037237248 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚425494664085504 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚429 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚209 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 845178880 PCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚58387380 _step‚1283 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚ 110085000 KBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚4876 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 831258624 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 3769790076416 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚57732047 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚426744588036258 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 4628521206 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 5494538240 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚4289 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚49026351 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚322 trainer/global_step‚15599 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚425494681190400 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 122905600 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 122905600 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚57732049 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚35942171 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 125829120 F?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚60 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4289 PIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚68 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17104896 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚41945088 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 5368709120ƒ „ïÁM2Ý$- _wandbruntime‚4142  _runtime‚4142¡tn8 Þ$ÈM A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚4295 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚49029440 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚425520986423296 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚ 110091840 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚322 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚423008853032960 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚425521003528192 IBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚60 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654602240 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 122905600 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 5494538240 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 125829120 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚4293 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚426771062789834 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚52356240 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 3770021145088 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚262 $ _timestamp‚1.786662794330568e+09 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚4293 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 H@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚665 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 125829120 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 3770021145088 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 3762168408806 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 5368709120 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚52360533 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚370948248076288 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚35944374 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 5494538240 K>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59053056 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚374935329804800 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚262 trainer/global_step‚15599 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 >5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5264 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚41948160 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚209 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large›½ø–_pool.peak‚ 5368709120 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 122306526 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚35944376 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 3770063093248 DDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 110096135 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚57735600 G@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚79 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚429291066621440 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 4654602240 I@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚4876 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚429 B8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚ 110091840 P6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚322 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚49029361 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚52360533 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 122905600 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚429291066621440 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚13085064 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 D=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚60 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚ 110096135 NADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚59053056 B9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚5264 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚57735602 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 3987081728512 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚322 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 4552982528 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚57735602 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚13084987 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521206 _step‚1284 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚425521003528192 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚374935400775168 SFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚70970368 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚370948268720128 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 G@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚60êr¬„Ó&ß$ÍM REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17104896 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 4552982528 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚52356240 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚41348068 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 5368709120 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚322 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚49029361 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚52360533 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 122905600 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚429291066621440 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚13085064 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 D=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚60 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚ 110096135 NADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚59053056 B9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚5264 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚57735602 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 3987081728512 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚322 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 4552982528 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚57735602 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚13084987 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521206 _step‚1284 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚425521003528192 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚374935400775168 SFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚70970368 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚370948268720128 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 G@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚60 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚4295 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚49029440 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚425520986423296 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚ 110091840 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚322 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚423008853032960 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚425521003528192 IBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚60 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654602240 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 122905600 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 5494538240 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 125829120 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚4293 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚426771062789834 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚52356240 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 3770021145088 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚262 $ _timestamp‚1.786662794330568e+09 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚4293 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 H@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚665 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 125829120 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 3770021145088 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 3762168408806 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 5368709120 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚52360533 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚370948248076288 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚35944374 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 5494538240 K>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59053056 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚374935329804800 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚262 trainer/global_step‚15599 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 >5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5264 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚41948160 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚209 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 5368709120 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 122306526 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚35944376 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 3770063093248 DDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 110096135 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚57735600 G@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚79 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚429291066621440 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 4654602240 I@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚4876 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚429 B8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚ 110091840 PDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4876 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17104896 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚427201718190080 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚13134269 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 4517085184 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 5368709120 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚57970303 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚4287 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚52540242 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚41940992 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚322 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 3777606966923 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚36093033 UHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚62584832 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚262 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 4654602240 JBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚665 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 125829120 D;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚5264 PIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚72 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚20643840 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 3777565618903 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚4287 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚36093035 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚262 JDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚57970305 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 trainer/global_step‚15649 _step‚1288 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚430987162756096 BDeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4289 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚52544529 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚49227230 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 5494538240 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 4628521206¿‚ùí$ÃO LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚262 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚41945088 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 3785444566016 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚428450463374987 @7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚5264 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚57970305 IBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚74 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚372376839651328 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 SDDeviceStatsMonitor.on_train_batch_start/reser¼ßS¬$ved_bytes.all.allocated‚ 5494538240 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚427201718190080 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 125829120 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚52544529 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 122905600 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚13134197 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 831258624 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚424672856408064 KDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚60 $ _timestamp‚1.786662811229721e+09 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚49227304 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 122905600 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚57970303 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚209 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚ 110510545 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚427201735294976 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚52540242 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚223 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚41945088 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 5494538240 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚430987221806080 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚428450404987607 F?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚60 PCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚59049984 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 3785486511104 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚ 110514834 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚61478912 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 122306526 _runtime‚4159.762871389 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 4654602240 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚427201735294976 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4289 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚52544529 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚49227230 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 5494538240 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 4628521206 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚322 G>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4876 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17104896 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚427201718190080 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚13134269 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 4517085184 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 5368709120 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚57970303 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚4287 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚52540242 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚41940992 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚322 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 3777606966923 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚36093033 UHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚62584832 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚262 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 4654602240 JBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚665 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 125829120 D;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚5264 PIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚72 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚20643840 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 3777565618903 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚4287 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚36093035 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚262 JDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚57970305 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 trainer/global_step‚15649 _step‚1288 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚430987162756096 B5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5264 P8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 5494538240 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚322 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚ 110517385 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚431013775215616 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 845178880 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚376403356203008 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚49230266 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚262 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 4552982528 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚44035072 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 5368709120 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚427228040527872 EDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 DDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59053056 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚49230332 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 125829120 G@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚60 B9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚5264 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚13135092 IBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚60 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 3785675634688 NGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚64 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚428476879741183 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚428476821353755 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚57973856 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚427228057632768 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚41948160 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚41348068 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚262 EDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521206 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 122905600 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚4293 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 3785717582848 SFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚64678912 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 125829120 _step‚1289 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚431013716162560 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654602240 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚52547822 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚52543529 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 122905600 NADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚58387428 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚4295 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚41948160 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 3785675634688 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚223 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 4517085184 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚57973858 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 $ _timestamp‚1.786662811390296e+09 _runtime‚4159.920680235 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 3785717582848 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚424699042201600 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚427228057632768 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚372399907667968 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 5368709120 D=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚60 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚322 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚429 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚57973858 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 4654602240 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚ 110517385 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 831258624 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 3777796191515 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 110521680 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚52547822 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚372399887024128 B6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚322 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17104896 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚209 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚427228040527872 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚52543529 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚262 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚36095240 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 4003469178880'ˆ€6Ó&ð$ÍM E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚429 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚57973858 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 4654602240 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚ 110517385 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 831258624 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 3777796191515 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 110521680 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚52547822 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚372399887024128 B6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚322 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17104896 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚209 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚427228040527872 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚52543529 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚262 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚36095240 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 4003469178880 >5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5264 P8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 5494538240 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚322 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚ 110517385 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚431013775215616 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 845178880 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚376403356203008 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚49230266 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚262 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 4552982528 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚44035072 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 5368709120 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚427228040527872 EDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 DDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59053056 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚49230332 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 125829120 G@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚60 B9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚5264 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚13135092 IBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚60 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 3785675634688 NGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚64 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚428476879741183 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚428476821353755 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚57973856 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚427228057632768 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚41948160 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚41348068 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚262 EDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521206 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 122905600 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚4293 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 3785717582848 SFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚64678912 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 125829120 _step‚1289 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚431013716162560 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654602240 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚52547822 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚52543529 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 122905600 NADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚58387428 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚4295 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚41948160 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 3785675634688 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚223 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 4517085184 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚57973858 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 $ _timestamp‚1.786662811390296e+09 _runtime‚4159.920680235 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 3785717582848 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚424699042201600 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚427228057632768 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚372399907667968 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 5368709120 D=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚60 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚322‰ zºu2ñ$- _wandbruntime‚4159  _runtime‚4159iÖQquò$ï 0loss/FRt1/assignment_loss‚0.4624166606136179 0loss/FRt2/assignment_loss‚0.3824390221728025 0loss/FRt1/detection_loss‚0.48251746260782685 &loss/total_loss‚1.8011743888561949 epoch‚39 trainer/global_step‚15649 _runtime‚ 4159.92131142 /loss/FRt2/detection_loss‚0.4738012434619475 % _timestamp‚1.7866628113914177e+09 _step‚1290ïßZzó$ô 0loss/FRt1/assignment_loss‚0.4624166606136179 0loss/FRt2/assignment_loss‚0.3824390221728025 0loss/FRt1/detection_loss‚0.48251746260782685 &loss/total_loss‚1.8011743888561949 epoch‚39 trainer/global_step‚15649 _runtime‚ 4159.92131142 /loss/FRt2/detection_loss‚0.4738012434619475 % _timestamp‚1.7866628113914177e+09 _step‚1290Š mB:¡ô$‚(:– ¥ŸùÓ·Œ¿á gpu.0.gpu‚64.0 gpu.0.memory‚12, gpu.0.memoryAllocated‚7.0313262939453125) gpu.0.memoryAllocatedBytes‚ 6039863296 gpu.0.temp‚35.0 gpu.0.powerWatts‚210.759' gpu.0.enforcedPowerLimitWatts‚500.0) gpu.0.powerPercent‚42.151799999999994 gpu.0.smClock‚1410 gpu.0.memoryClock‚1593$ gpu.0.correctedMemoryErrors‚1955# gpu.0.uncorrectedMemoryErrors‚0†N‚´œõ$‚(:‘ ¥ŸùӞ¼æ disk.sda3.in‚38394.64453125 disk.sda3.out‚ 76.83203125 disk.sda.in‚38394.64453125 disk.drbd2165.in‚0 disk.sda.out‚ 76.83203125 disk.sda2.in‚0! disk.nvme0n1.in‚ 44077.1640625& disk.nvme0n1.out‚60920.35205078125$ memory_percent‚68.26119422912598 cpu‚27.23656642629279# disk.nvme0c0n1.in‚ 44077.1640625( disk.nvme0c0n1.out‚60920.35205078125 network.recv‚ 7.852593e+06 disk.sda1.in‚0 disk.sda1.out‚0# proc.memory.rssMB‚ 2299.55078125) disk./.usagePercent‚32.69356120365977 disk.dm-1.in‚38394.64453125 disk.dm-1.out‚ 76.83203125 disk.sda2.out‚0( proc.memory.availableMB‚ 5200.0859375* proc.memory.percent‚14.035344123840332 proc.cpu.threads‚17 network.sent‚ 2.0415e+07% disk./.usageGB‚487.15768814086914 disk.drbd2165.out‚0 disk.dm-0.in‚ 44077.1640625# disk.dm-0.out‚60920.35205078125uš‚2ö$- _wandbruntime‚4172  _runtime‚4172øRlõÊ÷$Ä ' lr-AdamW/pg2‚0.00017127306638320004 trainer/global_step‚15699 $ _timestamp‚1.786662823611957e+09 _runtime‚4172.142101447 _step‚1291 ' lr-AdamW/pg1‚0.00017127306638320004çQOÏø$É $ _timestamp‚1.786662823611957e+09 _runtime‚4172.142101447 _step‚1291 ' lr-AdamW/pg1‚0.00017127306638320004 ' lr-AdamW/pg2‚0.00017127306638320004 trainer/global_step‚15699‹ 6§2ù$- _wandbruntime‚4172  _runtime‚41720Ø_xÄú$¿O N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚ 110857586 KBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚4876 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 122306526 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚322 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚432314866411520 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚58147955 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚41945088 JBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚665 IBDeviceStatsMonitor.on_train_batch_start/segment.small_6iº%pool.current‚60 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 4654602240 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 122905600 IBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚70 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚262 M@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚59049984 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚41945088 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚428517852184576 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 4517085184 KDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚60 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚61478912 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚4287 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚377544766605824 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚377544697729536 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 845178880 F?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚60 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 5368709120 @7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚5264 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚58147953 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 % _timestamp‚1.7866628236130278e+09 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 122905600 _runtime‚4172.146966221 PCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚59049984 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 3797073276928 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚58147955 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚52705344 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚373529228935168 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 4015537670656 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚322 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚428517835079680 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17104896 JDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 trainer/global_step‚15699 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚52709631 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 4628521206 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚ 110853297 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 3797073276928 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 831258624 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 5494538240 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 125829120 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚36203283 PIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚68 UHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚68876288 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚48232448 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚429771256475015 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 5494538240 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚36203285 C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚4289 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17104896 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚429 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 4552982528 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 3797031331840 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 3789168777531 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 3789127429511 _step‚1292 F>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 125829120 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 BDeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4289 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚20643840 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 4015489438208 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚49378275 PCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚58387380 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚432314925461504 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚13175060 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚209 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚262 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 125829120 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚322 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚432314925461504 G>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4876 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚428517852184576 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚52705344 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 3797031331840 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚428517835079680 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚223 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚425982146084864 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚425982129045504 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚432314866411520 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚49378345¹Ùñ?Ê'û$ÄO F>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 125829120 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 BDeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4289 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚20643840 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 4015489438208 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚49378275 PCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚58387380 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚432314925461504 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚13175060 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚209 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚262 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 125829120 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚322 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚432314925461504 G>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4876 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚428517852184576 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚52705344 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 3797031331840 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚428517835079680 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚223 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚425982146084864 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚425982129045504 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚432314866411520 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚49378345 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚ 110857586 KBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚4876 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 122306526 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚322 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚432314866411520 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚58147955 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚41945088 JBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚665 IBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚60 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 4654602240 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 122905600 IBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚70 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚262 M@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚59049984 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚41945088 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚428517852184576 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 4517085184 KDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚60 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚61478912 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚4287 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚377544766605824 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚377544697729536 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 845178880 F?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚60 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 5368709120 @7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚5264 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚58147953 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 % _timestamp‚1.7866628236130278e+09 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 122905600 _runtime‚4172.146966221 PCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚59049984 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 3797073276928 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚58147955 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚52705344 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚373529228935168 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 4015537670656 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚322 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚428517835079680 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17104896 JDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 trainer/global_step‚15699 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚52709631 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 4628521206 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚ 110853297 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 3797073276928 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 831258624 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 5494538240 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 125829120 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚36203283 PIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚68 UHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚68876288 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚48232448 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚429771256475015 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 5494538240 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚36203285 C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚4289 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17104896 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚429 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 4552982528 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 3797031331840 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 3789168777531 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 3789127429511 _step‚1292Œ ¯¤x2ü$- _wandbruntime‚4172  _runtime‚4172ÊZF¥Ï&ý$ÉM IBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚60 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 3797304348672 NADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚59053056 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚36205490 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 125829120 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚223 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚373552276307968 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 4517085184 G@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚60 trainer/global_step‚15699 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚432341419817984 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 3797304348672 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 3797262400512 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚13175798 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 D=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚60 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 4654602240 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚36205488 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚52708631 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚41948160 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚58151508 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚209 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚429797672841163 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 3789399350191 DDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚426008314839040 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 5368709120 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚262 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 110864432 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 4015738499584 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚432341478871040 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚428544174522368 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚322 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚4293 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚377567994163712 K>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59053056 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚377568056745472 SFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚62581760 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 845178880 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521206 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚262 % _timestamp‚1.7866628237758033e+09 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 3797262400512 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚428544157417472 B9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚5264 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 122306526 PDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 _step‚1293 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚4293 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚41348068 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 5368709120 _runtime‚4172.306205397 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚58151508 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 4552982528 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚373552255664128 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚322 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 G@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚75 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚49381286 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚428544157417472 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 831258624 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚41937920 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17104896 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654602240 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚49381361 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚61478912 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚52712924 H@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚665 NGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚73 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚4295 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚58151506 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 5494538240 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 4552982528 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚41948160 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚429 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 3789358002123 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 125829120 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 4015780437504 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚262 >5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5264 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚52708631 I@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚4876 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚20643840 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 B5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5264 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚52708631 I@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚4876 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚20643840 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 BDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚426008314839040 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 5368709120 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚262 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 110864432 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 4015738499584 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚432341478871040 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚428544174522368 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚322 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚4293 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚377567994163712 K>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59053056 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚377568056745472 SFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚62581760 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 845178880 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521206 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚262 % _timestamp‚1.7866628237758033e+09 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 3797262400512 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚428544157417472 B9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚5264 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 122306526 PDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 _step‚1293 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚4293 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚41348068 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 5368709120 _runtime‚4172.306205397 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚58151508 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 4552982528 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚373552255664128 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚322 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 G@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚75 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚49381286 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚428544157417472 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 831258624 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚41937920 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17104896 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654602240 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚49381361 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚61478912 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚52712924 H@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚665 NGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚73 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚4295 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚58151506 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 5494538240 xØ=22ÿ$- _wandbruntime‚4172  _runtime‚4172‚®Ó»x€%ò 0loss/FRt2/assignment_loss‚0.3593333617666076 0loss/FRt2/detection_loss‚0.47905556629184864 trainer/global_step‚15699 % _timestamp‚1.7866628237769809e+09 _step‚1294 1loss/FRt1/assignment_loss‚0.48664942748401385 0loss/FRt1/detection_loss‚0.49611167068356016 &loss/total_loss‚1.8211500262260303 epoch‚39 _runtime‚4172.306846986ܬ§A}%÷ 1loss/FRt1/assignment_loss‚0.48664942748401385 0loss/FRt1/detection_loss‚0.49611167068356016 &loss/total_loss‚1.8211500262260303 epoch‚39 _runtime‚4172.306846986 0loss/FRt2/assignment_loss‚0.3593333617666076 0loss/FRt2/detection_loss‚0.47905556629184864 trainer/global_step‚15699 % _timestamp‚1.7866628237769809e+09 _step‚1294Ž ÖrЪ2‚%- _wandbruntime‚4184  _runtime‚4184ˆU–ʃ%Ä _runtime‚4184.362876938 _step‚1295 ' lr-AdamW/pg1‚0.00016746849968021318 ' lr-AdamW/pg2‚0.00016746849968021318 trainer/global_step‚15749 $ _timestamp‚1.786662835831168e+09Ê~†HÏ„%É ' lr-AdamW/pg1‚0.00016746849968021318 ' lr-AdamW/pg2‚0.00016746849968021318 trainer/global_step‚15749 $ _timestamp‚1.786662835831168e+09 _runtime‚4184.362876938 _step‚1295 -³º°2…%- _wandbruntime‚4184  _runtime‚4184lBóqÄ'†%¾O [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 3808660042752 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚49529029 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚49528952 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚13215494 UHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚70973440 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚20643840 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 125829120 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚262 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 4552982528 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 4027773001216 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 4552982528 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 122905600 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 5368709120 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚262 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 845178880 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚50329600 F?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚60 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚429833951969280 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 4654602240 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 KDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚60 M@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚59049984 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 3808618097664 JDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚322 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 3800689240119 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚429 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚431092166349803 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚431092107962423 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚322 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚4287 IBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚60 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17104896 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚41945088 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 PIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚75 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚36313535 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚209 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 4027823330816 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 5494538240 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚262 G>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4876 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚52870446 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚61478912 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚433642570066944 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚433642570066944 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚13215419 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 831258624 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚322 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚41348020 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚36313533 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚429833969074176 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚223 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚58325603 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚ 111200338 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚378709349932544 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 5494538240 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 125829120 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚58325605 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 4654602240 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 3808618097664 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚58325605 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 5368709120 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 _runtime‚4184.362956879 _step‚1296 @7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚5264 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17104896 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4289 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚374681597575168 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚58325603 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 4628521206 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚433642629116928 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚374681576931328 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚322 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚427291418722304 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚52870446 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 5368709120 $ _timestamp‚1.786662835831822e+09¯¤É'‡%ÃO L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚52870446 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 5368709120 $ _timestamp‚1.786662835831822e+09 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 3808660042752 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚49529029 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚49528952 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚13215494 UHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚70973440 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚20643840 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 125829120 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚262 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 4552982528 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 4027773001216 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 4552982528 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 122905600 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 5368709120 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚262 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 845178880 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚50329600 F?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚60 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚429833951969280 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 4654602240 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 KDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚60 M@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚59049984 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 3808618097664 JDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚322 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 3800689240119 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚429 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚431092166349803 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚431092107962423 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚322 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚4287 IBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚60 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17104896 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚41945088 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 PIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚75 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚36313535 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚209 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 4027823330816 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 5494538240 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚262 G>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4876 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚52870446 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚61478912 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚433642570066944 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚433642570066944 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚13215419 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 831258624 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚322 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚41348020 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚36313533 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚429833969074176 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚223 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚58325603 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚ 111200338 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚378709349932544 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 5494538240 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 125829120 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚58325605 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 4654602240 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 3808618097664 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚58325605 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 5368709120 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 _runtime‚4184.362956879 _step‚1296 @7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚5264 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17104896 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4289 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚374681597575168 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚58325603 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 4628521206 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚433642629116928 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚374681576931328 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚322 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚427291418722304 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 ¯GöÎ2ˆ%- _wandbruntime‚4184  _runtime‚4184Óþl¶‰%ÉM E6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚322 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521206 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 122306526 IBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚60 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚41948160 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 111207184 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚49532043 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 845178880 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 125829120 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚322 _step‚1297 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 4654602240 H@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚665 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚49531976 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚13216303 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚433669182526464 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚429860291411968 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚378732646006272 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 831258624 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 3800919812731 PDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚429860291411968 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 3808891114496 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 122905600 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 NADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚58387428 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚41348068 % _timestamp‚1.7866628359932573e+09 >5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5264 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚374704644947968 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚20643840 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 5494538240 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 125829120 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚262 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚ 111207184 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚36315738 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚61478912 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚58329156 SFDeviceStatsMor4:pÉnitor.on_train_batch_end/inactive_split_bytes.all.current‚64678912 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚431118582715999 G@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚67 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚52878026 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚429860274307072 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 3808849166336 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚4293 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚429 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 5494538240 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17104896 B9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚5264 NGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚65 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚431118524328571 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚322 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 I@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚4876 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 5494538240 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚4293 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 125829120 trainer/global_step‚15749 K>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59053056 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 5368709120 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 4552982528 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 4517085184 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 5368709120 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 5368709120 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17104896 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 4552982528 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚52878026 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚427317604515840 D=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚60 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚262 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 3808849166336 EDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚52873733 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚223 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚433669182526464 NADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚59053056 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚433669123473408 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654602240 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚41948160 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚427317621555200 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚322 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚4295 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚58329158 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 B6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚322 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521206 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 122306526 IBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚60 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚41948160 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 111207184 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚49532043 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 845178880 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 125829120 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚322 _step‚1297 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 4654602240 H@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚665 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚49531976 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚13216303 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚433669182526464 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚429860291411968 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚378732646006272 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 831258624 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 3800919812731 PDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚429860291411968 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 3808891114496 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 122905600 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 NADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚58387428 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚41348068 % _timestamp‚1.7866628359932573e+09 >5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5264 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚374704644947968 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚20643840 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 5494538240 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 125829120 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚262 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚ 111207184 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚36315738 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚61478912 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚58329156 SFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚64678912 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚431118582715999 G@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚67 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚52878026 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚429860274307072 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 3808849166336 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚4293 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚429 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 5494538240 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17104896 B9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚5264 NGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚65 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚431118524328571 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚322 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 I@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚4876 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 5494538240 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚4293 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 125829120 trainer/global_step‚15749 K>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59053056 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 5368709120 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 4552982528 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 4517085184 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 5368709120 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 5368709120 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17104896 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 4552982528 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚52878026 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚427317604515840 D=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚60 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚262 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 3808849166336 EDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚52873733 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚223 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚433669182526464 NADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚59053056 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚433669123473408 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654602240 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚41948160 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚427317621555200 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚322 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚4295 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚58329158 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 BDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚434970273722368 D;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚5264 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 4040108005888 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚41348020 UHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚66779136 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 122306526 _step‚1300 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 122905600 @7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚5264 G>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4876 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚41945088 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚379874007441920 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚46135296 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚322 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚58503253 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 122905600 JDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚262 F>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚49680234 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚36423785 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚61478912 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚428600708399104 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 5368709120 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚434970332772352 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 4552982528 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 845178880 IBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚60 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚49680297 JDeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4289 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 % _timestamp‚1.7866628486150572e+09 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚41945088 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚58503253 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚262 _runtime‚4197.148871696 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚4287 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 5494538240 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 125829120 KDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚60 F?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚60 trainer/global_step‚15799 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚ 111543090 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚429 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚20643840 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚428600725438464 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 5494538240 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚431150068858880 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 3820246808576 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 PCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚58387380 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 4552982528 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 BDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4876 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚41945088 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚379874007441920 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚46135296 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚322 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚58503253 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 122905600 JDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚262 F>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚49680234 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚36423785 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚61478912 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚428600708399104 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 5368709120 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚434970332772352 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 4552982528p¼yMÖ SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 845178880 IBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚60 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚49680297 JDeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4289 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 % _timestamp‚1.7866628486150572e+09 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚41945088 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚58503253 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚262 _runtime‚4197.148871696 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚4287 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 5494538240 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 125829120 KDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚60 F?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚60 trainer/global_step‚15799 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚ 111543090 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚429 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚20643840 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚428600725438464 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 5494538240 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚431150068858880 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 3820246808576 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 PCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚58387380 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 4552982528 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 BDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚434970273722368 D;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚5264 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 4040108005888 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚41348020 UHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚66779136 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 122306526 _step‚1300 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 122905600 @7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚5264” ‹¶ýÒ2–%- _wandbruntime‚4197  _runtime‚4197‘gÏ&—%ÉM C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 5494538240 _runtime‚4197.307829247 EDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 D=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚60 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚434996886181888 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚431176391196672 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚434996827128832 E6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚322 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 125829120 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚ 111549936 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚53038835 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚49683287 G@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚65 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚36425990 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚375857013587968 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚262 % _timestamp‚1.7866628487774868e+09 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 I@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚4876 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 5494538240 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 125829120 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 3820477880320 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 3820435932160 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚53043128 NGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚63 SFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚62581760 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 5368709120 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚41948160 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚58506806 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 5368709120 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚262 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 3820435932160 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 122905600 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚49683222 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚61478912 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 4517085184 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 5494538240 DDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚36425988 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 4552982528 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚53038835 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚428626911232000 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚429 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚53043128 PDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654602240 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 4552982528 trainer/global_step‚15799 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚41348068 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚262 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 3812481623339 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚322 K>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59053056 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚13257297 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚209 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 4654602240 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17104896 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚41937920 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚431176408301568 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 122905600 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 111549936 NADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚58387428 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚432439375815979 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521206 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚20643840 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 4040309883392 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 _step‚1301 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚379897302827520 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚428626894192640 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚322 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚ 111545641 >5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5264d~fÚÔ&˜%ÎM E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 I@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚4876 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 5494538240 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 125829120 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 3820477880320 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 3820435932160 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚53043128 NGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚63 SFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚62581760 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 5368709120 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚41948160 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚58506806 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 5368709120 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚262 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 3820435932160 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 122905600 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚49683222 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚61478912 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 4517085184 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 5494538240 DDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚36425988 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 4552982528 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚53038835 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚428626911232000 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚429 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚53043128 PDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654602240 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 4552982528 trainer/global_step‚15799 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚41348068 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚262 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 3812481623339 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚322 K>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59053056 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚13257297 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚209 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 4654602240 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17104896 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚41937920 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚431176408301568 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 122905600 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 111549936 NADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚58387428 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚432439375815979 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521206 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚20643840 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 4040309883392 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 _step‚1301 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚379897302827520 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚428626894192640 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚322 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚ 111545641 >5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5264 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 5494538240 _runtime‚4197.307829247 EDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 D=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚60 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚434996886181888 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚431176391196672 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚434996827128832 E6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚322 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 125829120 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚ 111549936 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚53038835 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚49683287 G@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚65 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚36425990 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚375857013587968 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚262 % _timestamp‚1.7866628487774868e+09 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0• npi2™%- _wandbruntime‚4197  _runtime‚4197· œdwš%ñ epoch‚39 trainer/global_step‚15799 % _timestamp‚1.7866628487785027e+09 _step‚1302 1loss/FRt2/assignment_loss‚0.39364227091915416 /loss/FRt1/detection_loss‚0.5010994448167831 &loss/total_loss‚1.9032399303082983 _runtime‚4197.308318933 0loss/FRt1/assignment_loss‚0.5451704744290171 0loss/FRt2/detection_loss‚0.46332774014334394Í-ßÂ|›%ö 1loss/FRt2/assignment_loss‚0.39364227091915416 /loss/FRt1/detection_loss‚0.5010994448167831 &loss/total_loss‚1.9032399303082983 _runtime‚4197.308318933 0loss/FRt1/assignment_loss‚0.5451704744290171 0loss/FRt2/detection_loss‚0.46332774014334394 epoch‚39 trainer/global_step‚15799 % _timestamp‚1.7866628487785027e+09 _step‚1302– ÛZXê–œ%‚(:‹ ßùÓ­¥øà gpu.0.gpu‚62.0 gpu.0.memory‚11, gpu.0.memoryAllocated‚7.0313262939453125) gpu.0.memoryAllocatedBytes‚ 6039863296 gpu.0.temp‚34.0 gpu.0.powerWatts‚227.089' gpu.0.enforcedPowerLimitWatts‚500.0 gpu.0.powerPercent‚45.4178 gpu.0.smClock‚1410 gpu.0.memoryClock‚1593$ gpu.0.correctedMemoryErrors‚1955# gpu.0.uncorrectedMemoryErrors‚0±„ÊÕ…%‚(:ú ßùÓèõ—ä% disk.nvme0n1.out‚60939.3193359375$ disk./.usageGB‚487.1633529663086 disk.drbd2165.in‚0$ memory_percent‚68.26786994934082# proc.memory.rssMB‚ 2299.55078125* proc.memory.percent‚14.035344123840332 cpu‚27.235747900791303 disk.sda.in‚ 39528.625 network.sent‚ 2.0579155e+07 disk.sda3.in‚ 39528.625$ disk.nvme0c0n1.in‚44092.69921875 disk.sda1.in‚0" disk.nvme0n1.in‚44092.69921875 disk.dm-1.in‚ 39528.625 proc.cpu.threads‚17 network.recv‚ 7.910784e+06 disk.sda3.out‚77.3125 disk.sda.out‚77.3125 disk.sda2.out‚0 disk.dm-1.out‚77.3125( proc.memory.availableMB‚ 5198.9921875* disk./.usagePercent‚32.693941374848116 disk.drbd2165.out‚0 disk.dm-0.in‚44092.69921875" disk.dm-0.out‚60939.3193359375' disk.nvme0c0n1.out‚60939.3193359375 disk.sda1.out‚0 disk.sda2.in‚0MŸŽÝ2ž%- _wandbruntime‚4209  _runtime‚4209çûiÊŸ%Ä ' lr-AdamW/pg1‚0.00015997185296050978 ' lr-AdamW/pg2‚0.00015997185296050978 trainer/global_step‚15849 % _timestamp‚1.7866628612284372e+09 _runtime‚ 4209.75849035 _step‚13034¤Ë[Ï %É trainer/global_step‚15849 % _timestamp‚1.7866628612284372e+09 _runtime‚ 4209.75849035 _step‚1303 ' lr-AdamW/pg1‚0.00015997185296050978 ' lr-AdamW/pg2‚0.00015997185296050978— nQ²2¡%- _wandbruntime‚4209  _runtime‚4209éaª‘¶¢%¿O JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚4287 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚432466202853376 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚322 _step‚1304 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚223 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚58680903 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚432466185748480 JBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚665 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 4628521206 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 4517085184 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚41348020 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚53204937 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 3831791629312 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17104896 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 4552982528 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚ 111885842 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚429910015115264 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚436298036427776 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 3831791629312 PCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚59049984 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚53200650 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚433733810937239 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 F>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 M@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚59049984 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 4552982528 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚436297977377792 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚432466185748480 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 5494538240 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚262 KBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚4876 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚13297228 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 845178880 C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚4289 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 3831833574400 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 PCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚58387380 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 125829120 F?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚60 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 122905600 JDeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4289 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚209 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 4052399503360 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 3823854209355 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 3823812861335 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚322 % _timestamp‚1.7866628612294376e+09 D;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚5264 trainer/global_step‚15849 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17104896 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚49831263 UHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚66779136 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚381038667579392 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚20643840 G>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4876 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚436297977377792 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 5368709120 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 KDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚60 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚58680903 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 831258624 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚322 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚58680905 IBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚73 BDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚429910015115264 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚436298036427776 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 3831791629312 PCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚59049984 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚53200650 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚433733810937239 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 F>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 M@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚59049984 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 4552982528 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚436297977377792 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚432466185748480 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 5494538240 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚262 KBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚4876 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚13297228 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 845178880 C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚4289 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 3831833574400 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 PCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚58387380 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 125829120 F?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚60 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 122905600 JDeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4289 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚209 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 4052399503360 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 3823854209355 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 3823812861335 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚322 % _timestamp‚1.7866628612294376e+09 D;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚5264 trainer/global_step‚15849 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17104896 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚49831263 UHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚66779136 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚381038667579392 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚20643840 G>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4876 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚436297977377792 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 5368709120 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 KDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚60 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚58680903 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 831258624 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚322 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚58680905 IBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚73 B6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚322 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 3824084782015 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚322 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚49834215 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 831258624 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 5494538240 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚436324589837312 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 3832022697984 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 4052644531200 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚4295 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚53208230 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 122905600 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 845178880 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 5494538240 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 5368709120 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚262 G@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚60 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚ 111888393 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚223 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 _step‚1305 NADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚58387428 EDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59053056 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17104896 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 4552982528 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚53203937 SFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚66776064 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521206 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚429 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚36536240 DDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 111892688 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 3824043433947 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚41948160 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚4293 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚377009361584128 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚429936200908800 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚13298050 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚209 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚377009382227968 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 % _timestamp‚1.7866628613977034e+09 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚ 111892688 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 3832064646144 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚13297977 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚41348068 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚58684456 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 3832022697984 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚20643840 EDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654602240 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚49834290 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚46132224 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚436324589837312 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚433760285690815 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 122306526 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 125829120 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚ 111888393 >5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5264 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚58684458 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚432492525191168 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 5368709120 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚58684458 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚322 D=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚60 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 125829120 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚53203937 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 122905600 B9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚5264 G@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚75 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 125829120 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚432492508086272 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚322.=Ó¦%ÎM S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚433760227303387 IBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚60 trainer/global_step‚15849 _runtime‚4209.927977088 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚36536238 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚322 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 3824084782015 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚322 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚49834215 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 831258624 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 5494538240 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚436324589837312 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 3832022697984 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 4052644531200 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚4295 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚53208230 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 122905600 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 845178880 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 5494538240 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 5368709120 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚262 G@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚60 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚ 111888393 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚223 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 _step‚1305 NADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚58387428 EDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59053056 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17104896 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 4552982528 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚53203937 SFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚66776064 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521206 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚429 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚36536240 DDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 111892688 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 3824043433947 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚41948160 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚4293 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚377009361584128 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚429936200908800 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚13298050 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚209 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚377009382227968 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 % _timestamp‚1.7866628613977034e+09 H:DeviceStatsMonitor.on_train_batch_end/ac+‚×öÑ tive.all.allocated‚ 111892688 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 3832064646144 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚13297977 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚41348068 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚58684456 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 3832022697984 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚20643840 EDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654602240 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚49834290 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚46132224 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚436324589837312 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚433760285690815 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 122306526 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 125829120 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚ 111888393 >5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5264 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚58684458 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚432492525191168 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 5368709120 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚58684458 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚322 D=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚60 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 125829120 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚53203937 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 122905600 B9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚5264 G@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚75 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 125829120 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚432492508086272 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚322 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 BDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚437625681033216 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚53370039 C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚4289 G>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4876 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4289 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚58858555 KBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚4876 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚382203293409792 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 3835374671943 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 4654602240 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚437625681033216 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚49982007 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 5494538240 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17104896 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚433782319742976 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚322 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚262 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 122905600 D;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚5264 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚429 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚431219287752704 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 _step‚1308 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚53365752 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 4628521206 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 4654602240 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 4552982528 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 3835416019963 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 5494538240 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚437625740083200 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚433782302638080 PCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚59049984 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 5368709120 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚13337646 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 4517085184 trainer/global_step‚15899 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 4064610558464 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 F?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚60 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚53365752 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 122905600 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 KDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚60 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚223 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚435054720812027 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚ 112224305 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 125829120 % _timestamp‚1.7866628737359874e+09 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚53370039 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 3843420340224 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17104896 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚36644285 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 122306526 M@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚59049984 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚36644283 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚209 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚433782302638080 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 845178880 PCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚58387380 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚13337722 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚382203368577536 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚20643840 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚54523904 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 F>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 3843420340224 UHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚75167744 BDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 3843420340224 UHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚75167744 BDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚437625681033216 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚53370039 C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚4289 G>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4876 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4289 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚58858555 KBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚4876 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚382203293409792 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 3835374671943 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 4654602240 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚437625681033216 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚49982007 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 5494538240 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17104896 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚433782319742976 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚322 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚262 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 122905600 D;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚5264 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚429 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚431219287752704 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 _step‚1308 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚53365752 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 4628521206 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 4654602240 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 4552982528 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 3835416019963 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 5494538240 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚437625740083200 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚433782302638080œ hÛƵ2²%- _wandbruntime‚4222  _runtime‚4222[6O.ɳ%ÉM XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 831258624 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚435081137178223 K>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59053056 D=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚60 _runtime‚4222.431765411 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚13338540 SFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚68873216 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚48229376 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚435081078790795 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚58862106 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚41348068 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 3835605244555 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚58862108 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚53373332 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚433808624975872 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 122905600 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚382226659208704 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 5368709120 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 3843651411968 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 112235440 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚4293 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 EDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚49984967 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 125829120 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 5494538240 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 5494538240 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚322 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚437652234439680 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚433808642080768 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚ 112235440 E5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5264 D6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚322 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 5494538240 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚322 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 5368709120 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 B9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚5264 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 122306526 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚4293 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 4654602240 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 3843609463808 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚433808624975872 BDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521206 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 125829120 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 4552982528 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚49985030 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚4295 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654602240 H@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚665 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚209 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 4064860111360 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 4517085184 G@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚60 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚437652293492736 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚41948160 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚58862108 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚20643840 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 4064908340736 I@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚4876 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚429 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 5368709120 _step‚1309 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚61478912 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 NADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚58387428 G@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚63 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚58862106 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚378161750867968 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚53369039 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 3843651411968 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17104896 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 3843609463808 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 122905600 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 3835646592623 trainer/global_step‚15899 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚433808642080768 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚36646490 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚322 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚ 112231145 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚262 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚ 112231145 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚53369039 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚223FòËÔ&´%ÎM E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚58862106 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚378161750867968 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚53369039 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 3843651411968 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17104896 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 3843609463808 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 122905600 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 3835646592623 trainer/global_step‚15899 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚433808642080768 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚36646490 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚322 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚ 112231145 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚262 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚ 112231145 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚53369039 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚223 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 831258624 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚435081137178223 K>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59053056 D=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚60 _runtime‚4222.431765411 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚13338540 SFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚68873216 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚48229376 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚435081078790795 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚58862106 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚41348068 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 3835605244555 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚58862108 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚53373332 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚433808624975872 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 122905600 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚382226659208704 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 5368709120 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 3843651411968 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 112235440 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚4293 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 EDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚49984967 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 125829120 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 5494538240 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 5494538240 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚322 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚437652234439680 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚433808642080768 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚ 112235440 E5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5264 D6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚322 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 5494538240 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚322 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 5368709120 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 B9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚5264 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 122306526 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚4293 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 4654602240 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 3843609463808 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚433808624975872 BDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521206 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 125829120 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 4552982528 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚49985030 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚4295 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654602240 H@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚665 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚209 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 4064860111360 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 4517085184 G@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚60 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚437652293492736 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚41948160 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚58862108 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚20643840 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 4064908340736 I@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚4876 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚429 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 5368709120 _step‚1309 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚61478912 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 NADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚58387428 G@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚63 “¬¯2µ%- _wandbruntime‚4222  _runtime‚4222W]G—r¶%ì /loss/FRt1/detection_loss‚0.4917267324197425 /loss/FRt2/detection_loss‚0.4700456819522245 epoch‚39 % _timestamp‚1.7866628739024143e+09 _runtime‚ 4222.43228842 _step‚1310 /loss/FRt1/assignment_loss‚0.470674971298243 0loss/FRt2/assignment_loss‚0.3783269083968251 %loss/total_loss‚1.810774294067035 trainer/global_step‚15899à PTw·%ñ /loss/FRt1/detection_loss‚0.4917267324197425 /loss/FRt2/detection_loss‚0.4700456819522245 epoch‚39 % _timestamp‚1.7866628739024143e+09 _runtime‚ 4222.43228842 _step‚1310 /loss/FRt1/assignment_loss‚0.470674971298243 0loss/FRt2/assignment_loss‚0.3783269083968251 %loss/total_loss‚1.810774294067035 trainer/global_step‚15899ž m”–¸%‚(:‹ áŸùÓ¤û‹á gpu.0.gpu‚61.0 gpu.0.memory‚11, gpu.0.memoryAllocated‚7.0313262939453125) gpu.0.memoryAllocatedBytes‚ 6039863296 gpu.0.temp‚35.0 gpu.0.powerWatts‚145.754' gpu.0.enforcedPowerLimitWatts‚500.0 gpu.0.powerPercent‚29.1508 gpu.0.smClock‚1410 gpu.0.memoryClock‚1593$ gpu.0.correctedMemoryErrors‚1955# gpu.0.uncorrectedMemoryErrors‚0ââ_¾¹%‚(:„ áŸùÓäé‡ã disk.sda.out‚ 78.015625 disk.dm-1.out‚ 78.015625 disk.drbd2165.out‚0% disk./.usageGB‚487.16634368896484 disk.sda.in‚ 39528.625 disk.sda2.in‚0 disk.drbd2165.in‚0# proc.memory.rssMB‚ 2299.55078125* proc.memory.percent‚14.035344123840332 network.sent‚ 2.0718592e+07 disk.sda1.in‚0 disk.sda2.out‚0" disk.nvme0n1.in‚44092.69921875& disk.nvme0n1.out‚60950.55419921875 disk.dm-1.in‚ 39528.625# disk.dm-0.out‚60950.55419921875 cpu‚27.23486702965083 disk.sda3.out‚ 78.015625( disk.nvme0c0n1.out‚60950.55419921875 disk.sda1.out‚0 disk.dm-0.in‚44092.69921875$ memory_percent‚68.26703548431396) proc.memory.availableMB‚ 5199.12890625 proc.cpu.threads‚17 network.recv‚ 7.963991e+06* disk./.usagePercent‚32.694142084754965 disk.sda3.in‚ 39528.625$ disk.nvme0c0n1.in‚44092.69921875I< ÿ2º%- _wandbruntime‚4234  _runtime‚4234à»TË»%Å ' lr-AdamW/pg1‚0.00015262678942809846 ' lr-AdamW/pg2‚0.00015262678942809846 trainer/global_step‚15949 % _timestamp‚1.7866628863703299e+09 _runtime‚4234.900339629 _step‚1311“ßGÊм%Ê ' lr-AdamW/pg2‚0.00015262678942809846 trainer/global_step‚15949 % _timestamp‚1.7866628863703299e+09 _runtime‚4234.900339629 _step‚1311 ' lr-AdamW/pg1‚0.00015262678942809846Ÿ ²ýÊå2½%- _wandbruntime‚4234  _runtime‚4234 |AØÅ'¾%¿O PCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚59049984 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚48232448 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 3846936482551 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚59036203 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚53530854 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚379291051491328 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 F>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 G>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4876 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚438953443738624 D;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚5264 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚383368011560960 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 5368709120 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚61478912 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚59036205 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚36754533 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚41348020 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚438953443738624 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 5494538240 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚50132795 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚432528594468864 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 125829120 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚262 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚13378334 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚262 F?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚60 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4289 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚209 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 JBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚665 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚322 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 4552982528 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚432528577429504 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 4654602240 KBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚4876 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚436375513912055 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚435098419527680 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚41945088 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 5494538240 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 125829120 C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚4289 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚53530854 UHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚68876288 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 3855007106048 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17104896 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚429 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 4076939425792 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 122306526 IBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚60 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚223 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚436375572299435 PCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚58387380 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 KDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚60 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚53535141 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚438953384688640 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚379291072135168 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚20643840 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 3846977830571 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 5368709120 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚438953384688640 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚36754535 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 BDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 125829120 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 _step‚1312 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚53535141 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚59036205 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17104896 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚435098436632576 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚41945088 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 3854965160960 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 831258624 _runtime‚4234.901595201 @7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚5264 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 4654602240 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚435098419527680 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚322 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚262 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 122905600 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚59036203 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚13378262 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 122905600 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚4287 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚383367942684672 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 5494538240 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 5368709120 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚435098436632576 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 3854965160960 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 4517085184 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 % _timestamp‚1.7866628863712962e+09 M@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚59049984KS¢Ç¿%ÄO C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚4289 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚53530854 UHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚68876288 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 3855007106048 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17104896 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚429 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 4076939425792 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 122306526 IBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚60 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚223 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚436375572299435 PCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚58387380 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 KDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚60 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚53535141 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚438953384688640 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚379291072135168 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚20643840 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 3846977830571 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 5368709120 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚438953384688640 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚36754535 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 BDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 125829120 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 _step‚1312 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚53535141 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚59036205 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17104896 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚435098436632576 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚41945088 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 3854965160960 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 831258624 _runtime‚4234.901595201 @7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚5264 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 4654602240 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚435098419527680 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚322 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚262 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 122905600 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚59036203 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚13378262 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 122905600 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚4287 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚383367942684672 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 5494538240 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 5368709120 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚435098436632576 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 3854965160960 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 4517085184 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 % _timestamp‚1.7866628863712962e+09 M@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚59049984 PCDeviceStatsMo`3;nitor.on_train_batch_start/allocated_bytes.all.current‚59049984 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚48232448 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 3846936482551 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚59036203 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚53530854 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚379291051491328 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 F>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 G>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4876 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚438953443738624 D;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚5264 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚383368011560960 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 5368709120 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚61478912 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚59036205 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚36754533 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚41348020 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚438953443738624 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 5494538240 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚50132795 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚432528594468864 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 125829120 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚262 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚13378334 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚262 F?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚60 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4289 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚209 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 JBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚665 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚322 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 4552982528 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚432528577429504 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 4654602240 KBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚4876 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚436375513912055 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚435098419527680 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚41945088 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 5494538240 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 125829120  ;ô2À%- _wandbruntime‚4235  _runtime‚4235“`ŒÏ&Á%ÉM UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚41948160 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚4293 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 4077186190336 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚322 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 5368709120 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚59039756 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521206 G@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚60 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚435124741865472 B9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚5264 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚20643840 P8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚322 DDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 112578192 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚36756740 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚438979997148160 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 3847167055163 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 5368709120 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚322 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚4293 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 122905600 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚ 112573897 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚53538434 I@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚4876 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 EDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚53538434 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚13379102 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚209 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 NGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚66 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚379314119507968 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 BDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚50135774 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚41948160 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚438979938095104 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚435124758970368 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17104896 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚432554780262400 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚61478912 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚262 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚262 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 4654602240 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚59039756 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚379314098864128 >5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5264 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 4552982528 EDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59053056 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚383391234727936 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 5494538240 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚322 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚4295 H@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚665 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚13379036 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 122306526 _step‚1313 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚53534141 SFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚70970368 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654602240 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚59039758 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚ 112573897 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 4552982528 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚223 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 4077135863808 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 5368709120 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚438979997148160 NADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚59053056 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚53534141 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0rƒ‚Ô&Â%ÎM NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚53538434 I@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚4876 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 EDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚53538434 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚13379102 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚209 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 NGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚66 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚379314119507968 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 BDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚50135774 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚41948160 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚438979938095104 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚435124758970368 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17104896 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚432554780262400 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚61478912 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚262 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚262 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 4654602240 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚59039756 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚379314098864128 >5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5264 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 4552982528 EDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59053056 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚383391234727936 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 5494538240 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚322 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚4295 H@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚665 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚13379036 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 122306526 _step‚1313 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚53534141 SFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚70970368 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654602240 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚59039758 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚ 112573897 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 4552982528 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚223 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 4077135863808 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 5368709120 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚438979997148160 NADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚59053056 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚53534141 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚41948160 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚4293 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 4077186190336 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚322 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 5368709120 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚59039756 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521206 G@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚60 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚435124741865472 B9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚5264 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚20643840 P8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚322 DDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 112578192 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚36756740 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚438979997148160 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 3847167055163 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 5368709120 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚322 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚4293 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 122905600 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚ 112573897 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360¡ el¢¾2Ã%- _wandbruntime‚4235  _runtime‚42354Å¿xuÄ%ï trainer/global_step‚15949 % _timestamp‚1.7866628865356944e+09 _runtime‚ 4235.06550514 0loss/FRt1/assignment_loss‚0.5010908536572355 &loss/total_loss‚1.8355036383076473 epoch‚39 _step‚1314 1loss/FRt2/assignment_loss‚0.38758276152677046 /loss/FRt1/detection_loss‚0.5056006414793371 /loss/FRt2/detection_loss‚0.44122938164430415•»mzÅ%ô 0loss/FRt1/assignment_loss‚0.5010908536572355 &loss/total_loss‚1.8355036383076473 epoch‚39 _step‚1314 1loss/FRt2/assignment_loss‚0.38758276152677046 /loss/FRt1/detection_loss‚0.5056006414793371 /loss/FRt2/detection_loss‚0.4412293816443041 trainer/global_step‚15949 % _timestamp‚1.7866628865356944e+09 _runtime‚ 4235.06550514¢ lRÄ£–Æ%‚(:‹ ðŸùÓ˜Ñéà gpu.0.gpu‚64.0 gpu.0.memory‚12, gpu.0.memoryAllocated‚7.0313262939453125) gpu.0.memoryAllocatedBytes‚ 6039863296 gpu.0.temp‚35.0 gpu.0.powerWatts‚196.577' gpu.0.enforcedPowerLimitWatts‚500.0 gpu.0.powerPercent‚39.3154 gpu.0.smClock‚1410 gpu.0.memoryClock‚1593$ gpu.0.correctedMemoryErrors‚1955# gpu.0.uncorrectedMemoryErrors‚0ÜÈÁC“Ç%‚(:ˆ ðŸùÓÚºâ) proc.memory.availableMB‚ 5199.34765625# proc.memory.rssMB‚ 2299.55078125* proc.memory.percent‚14.035344123840332) disk./.usagePercent‚32.69415232505633$ disk.nvme0c0n1.in‚44099.55859375 disk.sda1.in‚0" disk.nvme0n1.in‚44099.55859375 disk.drbd2165.in‚0 disk.sda3.out‚ 78.33984375 disk.sda.out‚ 78.33984375 disk.sda1.out‚0 disk.sda2.out‚0" memory_percent‚68.265700340271 cpu‚27.234519249160908 proc.cpu.threads‚17 network.recv‚ 7.988373e+06 disk.sda3.in‚ 39528.625 disk.sda.in‚ 39528.625& disk.nvme0n1.out‚60958.04345703125 disk.dm-1.in‚ 39528.625 disk.dm-1.out‚ 78.33984375 disk.drbd2165.out‚0 disk.dm-0.in‚44099.55859375 network.sent‚ 2.0786719e+07% disk./.usageGB‚487.16649627685547( disk.nvme0c0n1.out‚60958.04345703125 disk.sda2.in‚0# disk.dm-0.out‚60958.043457031250qºW2È%- _wandbruntime‚4247  _runtime‚4247›^S ÉÉ%à & lr-AdamW/pg1‚0.0001490116925818256 & lr-AdamW/pg2‚0.0001490116925818256 trainer/global_step‚15999 % _timestamp‚1.7866628987846932e+09 _runtime‚4247.314732611 _step‚1315\ííëÎÊ%È _step‚1315 & lr-AdamW/pg1‚0.0001490116925818256 & lr-AdamW/pg2‚0.0001490116925818256 trainer/global_step‚15999 % _timestamp‚1.7866628987846932e+09 _runtime‚4247.314732611£ D#2Ë%- _wandbruntime‚4247  _runtime‚42470#îûÌ%¿O ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 3866551926784 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚59213853 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚322 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚322 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚436414536417280 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚384532588216832 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚61478912 C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚4289 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17104896 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 122905600 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚436414536417280 IBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚69 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚223 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 5494538240 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚433837884145664 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚433837867106304 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 5368709120 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚436414553522176 PCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚59049984 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚ 112914098 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚380443440775168 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 4654602240 JDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚440281088344064 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 3866593871872 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚262 trainer/global_step‚15999 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚59213853 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4289 PIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚67 UHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚64681984 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 % _timestamp‚1.7866628987856295e+09 _step‚1316 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚ 112909809 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚20643840 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚53700243 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚13419156 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 3858539641179 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 5368709120 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚440281147394048 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 KBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚4876 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚437696423786843 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚437696365399463 G>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4876 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 4628521206 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚53695956 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 122905600 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 845178880 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 4552982528 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 _runtime‚4247.315893893 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚380443420131328 HBDeviceStatsMonitor.on_train_batÏDch_start/oversize_allocations.freed‚0 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚262 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 4089168085504 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚53695956 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 125829120 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚41945088 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚59213855 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚44038144 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 PCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚58387380 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 4517085184 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚440281088344064 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚429 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 122306526 IBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚60 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚4287 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 4089212123648 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚50284008 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚13419223 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚36864783 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚4287 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚440281147394048 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚53700243 KDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚60 @7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚5264 F>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚322 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚436414553522176 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17104896 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚209 F?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚60 JDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚440281088344064 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 3866593871872 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚262 trainer/global_step‚15999 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚59213853 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4289 PIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚67 UHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚64681984 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 % _timestamp‚1.7866628987856295e+09 _step‚1316 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚ 112909809 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚20643840 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚53700243 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚13419156 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 3858539641179 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 5368709120 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚440281147394048 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 KBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚4876 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚437696423786843 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚437696365399463 G>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4876 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 4628521206 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚53695956 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 122905600 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 845178880 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 4552982528 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 _runtime‚4247.315893893 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚380443420131328 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚262 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 4089168085504 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚53695956 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 125829120 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚41945088 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚59213855 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚44038144 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 PCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚58387380 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 4517085184 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚440281088344064 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚429 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 122306526 IBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚60 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚4287 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 4089212123648 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚50284008 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚13419223 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚36864783 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚4287 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚440281147394048 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚53700243 KDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚60 @7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚5264 F>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚322 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚436414553522176 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17104896 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚209 F?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚60 JDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚440307700803584 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 3866782995456 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 5494538240 D6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚322 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚384555946675712 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 _runtime‚4247.478819499 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚ 112920944 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17104896 H@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚665 BDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654602240 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚384555875705344 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 5494538240 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17104896 I@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚4876 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚429 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 5368709120 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚322 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚53703536 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 3866824943616 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 112920944 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚20643840 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚433864069939200 % _timestamp‚1.7866628989485118e+09 G@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚71 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521206 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 125829120 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚53699243 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚ 112916649 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚59217408 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 831258624 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚41348068 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚262 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚13420051 NGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚69 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚13419982 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 _step‚1317 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 4089408201216 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 D=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚60 PDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 5368709120 IBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚60 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚59217408 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 122905600 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚209 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚4295 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚ 112916649 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚436440858755072 NADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚59053056 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚436440875859968 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚4293 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚53699243 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚4293 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 4654602240 B9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚5264 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚262 trainer/global_step‚15999 E6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚322 EDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚50286970 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚380466488147968 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 125829120 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 122905600 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 3858770213839 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 5494538240 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 3866824943616 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 4517085184 G@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚60 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 4552982528 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚36866990 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 3858728865771 K>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59053056 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 845178880 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 4089458527744 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚436440858755072 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚437722840153039 NADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚58387428 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚262 >5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5264 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚59217406 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚440307700803584 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚36866988 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚437722781765611 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 122306526 SFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚70970368 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚50326528 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚433864052899840 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0Ž3Á,Î Ð%ÎM D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 5368709120 IBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚60 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚59217408 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 122905600 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚209 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚4295 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚ 112916649 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚436440858755072 NADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚59053056 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚436440875859968 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚4293 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚53699243 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚4293 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 4654602240 B9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚5264 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚262 trainer/global_step‚15999 E6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚322 EDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚50286970 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚380466488147968 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 125829120 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 122905600 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 3858770213839 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 5494538240 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 3866824943616 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 4517085184 G@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚60 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 4552982528 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚36866990 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 3858728865771 K>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59053056 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 845178880 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 4089458527744 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚436440858755072 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚437722840153039 NADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚58387428 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚262 >5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5264 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚59217406 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚440307700803584 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚36866988 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚437722781765611 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 122306526 SFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚70970368 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚50326528 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚433864052899840 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚223 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚41948160 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚440307700803584 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 3866782995456 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 5494538240 D6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚322 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚384555946675712 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 _runtime‚4247.478819499 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚ 112920944 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17104896 H@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚665 BDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654602240 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚384555875705344 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 5494538240 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17104896 I@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚4876 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚429 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 5368709120 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚322 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚53703536 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 3866824943616 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 112920944 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚20643840 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚433864069939200 % _timestamp‚1.7866628989485118e+09 G@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚71 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 46285212àN06 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 125829120 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚53699243 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚ 112916649 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚59217408 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 831258624 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚41348068 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚262 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚13420051 NGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚69 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚13419982 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 _step‚1317 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 4089408201216 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 D=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚60 PDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4876 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 3874167423888 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚41348020 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚59452111 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚59452109 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚322 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚37015649 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 5494538240 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 5494538240 _step‚1321 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚442003796938240 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 4654602240 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚429 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚223 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 5368709120 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚53883245 IBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚73 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚381895059079168 JDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 4628521206 F>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 PCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚59049984 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 3882248361472 BDeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4289 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 3882248361472 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 4654602240 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚438121590521856 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 4105577675776 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 125829120 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚50485491 M@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚59049984 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚438121607626752 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚442003855988224 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17104896 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 3874126075868 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 5368709120 trainer/global_step‚16049 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚381895079723008 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 4552982528 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚61478912 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚439402182350812 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚ 113339643 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚50485418 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚ 113335354 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 4552982528 PIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚71 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 845178880 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚435528056274944 _runtime‚ 4265.02687064 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚4287 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚442003796938240 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 122905600 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚13469842 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚41945088 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚59452111 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚20643840 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚322 % _timestamp‚1.7866629164933999e+09 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚53883245 PCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚58387380 KDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚60 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚59452109 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0».iÉ'Þ%ÃO WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 3882206416384 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚262 @7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚5264 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 3882206416384 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚386000657398784 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚53887532 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚438121590521856 UHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚73070592 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 125829120 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 122905600 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚209 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚386000584328192 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 4105525249024 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚439402240738192 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚438121607626752 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚442003855988224 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4289 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 3882248361472 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 4654602240 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚438121590521856 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 4105577675776 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 125829120 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚50485491 M@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚59049984 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚438121607626752 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚442003855988224 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17104896 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 3874126075868 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 5368709120 trainer/global_step‚16049 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚381895079723008 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 4552982528 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚61478912 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚439402182350812 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚ 113339643 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚50485418 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚ 113335354 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 4552982528 PIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚71 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 845178880 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚435528056274944 _runtime‚ 4265.02687064 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚4287 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚442003796938240 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 122905600 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚13469842 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚41945088 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚59452111 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚20643840 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚322 % _timestamp‚1.7866629164933999e+09 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚53883245 PCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚58387380 KDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚60 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚59452109 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 JDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4876 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 3874167423888 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚41348020 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚59452111 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚59452109 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚322 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚37015649 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 5494538240 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 5494538240 _step‚1321 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚442003796938240 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 4654602240 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚429 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚223 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 5368709120 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚53883245 IBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚73 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚381895059079168 JDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 4628521206 F>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 PCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚59049984 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 3882248361472 B8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚4293 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚37017852 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 831258624 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 3874356648480 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚59455662 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚37017854 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚59455662 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚13470636 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚429 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚322 _step‚1322 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚41948160 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚ 113342194 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚41348068 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚262 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚442030409397760 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17104896 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚4293 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚44035072 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 845178880 E6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚322 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 4552982528 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚41948160 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚262 % _timestamp‚1.7866629166619544e+09 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚53886532 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚442030409397760 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚59455664 trainer/global_step‚16049 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654602240 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 113346489 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521206 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 5368709120 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 IBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚60 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 K>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59053056 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 5368709120 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚59455664 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17104896 NADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚59053056 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚53886532 I@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚4876 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚50488433 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚20643840 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚53890825 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚438147929964544 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 4105772049408 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 D>DeviceStatsMoniUÞÇa2tor.on_train_batch_end/segment.small_pool.freed‚0 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 4105816084480 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 5368709120 >5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5264 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 5494538240 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚61478912 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 G@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚60 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚4295 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚438147912859648 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 4552982528 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 122905600 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚386023878501376 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 125829120 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚ 113346489 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 122905600 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚442030350344704 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚435554242068480 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 5494538240 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚262 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚381918127095808 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 G@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚57 D=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚60 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 3882437485056 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚13470581 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 3874397996548 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 5494538240 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 125829120 _runtime‚4265.192376614 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚386023943180288 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 4517085184 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 122306526 P6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚322 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 3882437485056 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚53890825 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚322 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 4654602240 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 3882479433216 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚223‘Ó#"Ô&á%ÎM K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚50488433 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚20643840 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚53890825 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚438147929964544 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 4105772049408 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 4105816084480 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 5368709120 >5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5264 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 5494538240 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚61478912 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 G@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚60 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚4295 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚438147912859648 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 4552982528 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 122905600 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚386023878501376 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 125829120 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚ 113346489 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 122905600 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚442030350344704 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚435554242068480 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 5494538240 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚262 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚381918127095808 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 G@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚57 D=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚60 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 3882437485056 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚13470581 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 3874397996548 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 5494538240 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 125829120 _runtime‚4265.192376614 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚386023943180288 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 4517085184 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 122306526 P6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚322 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 3882437485056 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚53890825 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚322 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 4654602240 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 3882479433216 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚223 SFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚64678912 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 B8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚4293 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚37017852 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 831258624 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 3874356648480 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚59455662 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚37017854 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚59455662 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚13470636 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚429 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚322 _step‚1322 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚41948160 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚ 113342194 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚41348068 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚262 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚442030409397760 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17104896 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚4293 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚44035072 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 845178880 E6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚322 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 4552982528 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚41948160 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚262 % _timestamp‚1.7866629166619544e+09 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚53886532 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚442030409397760 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚59455664 trainer/global_step‚16049 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654602240 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 113346489 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521206 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 5368709120 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 IBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚60 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 K>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59053056 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 5368709120 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚59455664 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17104896 NADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚59053056 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚53886532 I@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚4876ª ­Ž +2â%- _wandbruntime‚4265  _runtime‚4265„þÃJwã%ñ 1loss/FRt1/assignment_loss‚0.43726458310177424 0loss/FRt2/assignment_loss‚0.3694444576617341 epoch‚40 trainer/global_step‚16049 % _timestamp‚1.7866629166631918e+09 _runtime‚4265.193193127 /loss/FRt1/detection_loss‚0.4508092072468257 0loss/FRt2/detection_loss‚0.43425341278451257 &loss/total_loss‚1.6917716607948468 _step‚1323 V- |ä%ö _step‚1323 1loss/FRt1/assignment_loss‚0.43726458310177424 0loss/FRt2/assignment_loss‚0.3694444576617341 epoch‚40 trainer/global_step‚16049 % _timestamp‚1.7866629166631918e+09 _runtime‚4265.193193127 /loss/FRt1/detection_loss‚0.4508092072468257 0loss/FRt2/detection_loss‚0.43425341278451257 &loss/total_loss‚1.6917716607948468« ¨£ K¡å%‚(:– Ž ùÓçÒ‰á gpu.0.gpu‚64.0 gpu.0.memory‚12, gpu.0.memoryAllocated‚7.0313262939453125) gpu.0.memoryAllocatedBytes‚ 6039863296 gpu.0.temp‚36.0 gpu.0.powerWatts‚211.046' gpu.0.enforcedPowerLimitWatts‚500.0) gpu.0.powerPercent‚42.209199999999996 gpu.0.smClock‚1410 gpu.0.memoryClock‚1593$ gpu.0.correctedMemoryErrors‚1955# gpu.0.uncorrectedMemoryErrors‚0uÁwæ%‚(:ì Ž ùÓÉ¿Ûâ disk.sda3.out‚ 79.1328125 disk.sda.out‚ 79.1328125 disk.sda2.in‚0 disk.sda2.out‚0" disk.nvme0n1.out‚ 61011.3515625 network.recv‚ 8.043443e+06 disk.dm-0.in‚ 44101.21875$ disk.nvme0c0n1.out‚ 61011.3515625 disk.sda1.in‚0 disk.sda1.out‚0 disk.dm-1.in‚ 39528.625 disk.drbd2165.in‚0 cpu‚27.23931989278103& proc.memory.availableMB‚ 5198.59375# proc.memory.rssMB‚ 2299.55078125* proc.memory.percent‚14.035344123840332 proc.cpu.threads‚17 network.sent‚ 2.092883e+07) disk./.usagePercent‚32.69834649648979! disk.nvme0c0n1.in‚ 44101.21875 disk.sda.in‚ 39528.625 disk.nvme0n1.in‚ 44101.21875 disk.dm-1.out‚ 79.1328125 disk.drbd2165.out‚0$ memory_percent‚68.27030181884766$ disk./.usageGB‚487.2289924621582 disk.dm-0.out‚ 61011.3515625 disk.sda3.in‚ 39528.625N² V2ç%- _wandbruntime‚4277  _runtime‚4277¯æ¦Êè%Ä _runtime‚ 4277.43310843 _step‚1324 ' lr-AdamW/pg1‚0.00014189752966880818 ' lr-AdamW/pg2‚0.00014189752966880818 trainer/global_step‚16099 % _timestamp‚1.7866629289030862e+09%TœsÏé%É ' lr-AdamW/pg1‚0.00014189752966880818 ' lr-AdamW/pg2‚0.00014189752966880818 trainer/global_step‚16099 % _timestamp‚1.7866629289030862e+09 _runtime‚ 4277.43310843 _step‚1324¬ ÌFF(2ê%- _wandbruntime‚4277  _runtime‚4277)?™iÅ'ë%¿O JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 122306526 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚322 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚ 113678106 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17104896 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚50636238 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 4117792774144 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚322 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 3893835127296 KBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚4876 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚436837362991104 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 4517085184 KDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚60 F?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚60 C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚4289 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚4287 UHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚64681984 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚209 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 5368709120 F>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4289 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 125829120 M@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚59049984 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 4552982528 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚41945088 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚383047448363008 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 5494538240 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 5494538240 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 G>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4876 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 122905600 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚387165285175296 _runtime‚4277.434286918 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 4628521206 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚54052634 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚59629759 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17104896 JDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚443331500593664 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚41945088 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚223 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚13510339 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 831258624 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 4117836812288 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚54052634 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚443331500593664 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚429 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 5368709120 trainer/global_step‚16099 IBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚65 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚322 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚262 % _timestamp‚1.7866629289039915e+09 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚59629759 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 5368709120 PIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚63 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚440723092225600 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚262 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚ 113682395 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚59629761 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 4552982528 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚13510276 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 845178880 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚59629761 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 4654602240 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚20643840 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚443331559643648 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚50636173 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚443331559643648 PCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚59049984 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 3893793182208 PCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚58387380 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚262 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚439437707411456 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 122905600 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚61478912 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 3885729234496 JDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚443331500593664 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚41945088 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚223 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚13510339 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 831258624 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 4117836812288 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚54052634 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚443331500593664 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚429 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 5368709120 trainer/global_step‚16099 IBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚65 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚322 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚262 % _timestamp‚1.7866629289039915e+09 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚59629759 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 5368709120 PIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚63 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚440723092225600 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚262 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚ 113682395 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚59629761 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 4552982528 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚13510276 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 845178880 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚59629761 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 4654602240 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚20643840 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚443331559643648 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚50636173 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚443331559643648 PCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚59049984 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 3893793182208 PCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚58387380 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚262 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚439437707411456 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 122905600 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚61478912 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 3885729234496 JDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4289 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 125829120 M@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚59049984 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 4552982528 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚41945088 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚383047448363008 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 5494538240 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 5494538240 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 G>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4876 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 122905600 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚387165285175296 _runtime‚4277.434286918 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 4628521206 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚54052634 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚59629759 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17104896 JDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 # _timestamp‚1.78666292906416e+09 PDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚50639284 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚383070475091968 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚59633314 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 4118035347456 B8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚436863548784640 EDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654602240 SFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚66776064 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚41348068 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 125829120 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚262 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚439464046854144 I@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚4876 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚46132224 K>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59053056 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 4552982528 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚41948160 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 5368709120 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚262 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚54051634 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚443358113053184 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 3894066199040 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚209 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚383070495735808 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚61478912 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚4293 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚4293 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚20643840 NADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚58387428 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚322 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 3894024250880 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 122905600 G@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚67 NGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚65 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚262 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚ 113684946 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚443358054000128 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 5494538240 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚322 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 4118081479680 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚440749450204368 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚322 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 122306526 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 125829120 >5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5264 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚37128102 _runtime‚4277.594362917 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 5368709120 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚13511180 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚387188577215488 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 125829120 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚50639217 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚429 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚59633312 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 122905600 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521206 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚436863531745280 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 4517085184 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚223 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚54055927 IBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚60 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 4654602240 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚439464046854144 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚54055927 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚443358113053184 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚13511115 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚322 _step‚1326 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚439464029749248 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 113689241 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 3885918459088 DDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 NADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚59053056 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚37128104 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚4295 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 3894024250880 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚59633312 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 845178880 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚440749508591796 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 4552982528 EDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 # _timestamp‚1.78666292906416e+09 PDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚50639284 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚383070475091968 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚59633314 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 4118035347456 B8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚436863548784640 EDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654602240 SFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚66776064 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚41348068 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 125829120 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚262 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚439464046854144 I@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚4876 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚46132224 K>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59053056 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 4552982528 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚41948160 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 5368709120 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚262 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚54051634 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚443358113053184 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 3894066199040 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚209 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚383070495735808 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚61478912 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚4293 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚4293 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚20643840 NADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚58387428 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚322 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 3894024250880 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 122905600 G@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚67 NGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚65 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚262 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚ 113684946 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚443358054000128 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 5494538240 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚322 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 4118081479680 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚440749450204368 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚322 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 122306526 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 125829120 >5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5264 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚37128102 _runtime‚4277.594362917 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 5368709120 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚13511180 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚387188577215488 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 125829120 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚50639217 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚429 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚59633312 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 122905600 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521206 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚436863531745280 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 4517085184 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚223 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚54055927 IBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚60 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 4654602240 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚439464046854144 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚54055927 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚443358113053184 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚13511115 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚322 _step‚1326 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚439464029749248 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 113689241 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 3885918459088 DDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 NADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚59053056 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚37128104 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚4295 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 3894024250880 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚59633312 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 845178880 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚440749508591796 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 4552982528 EDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 122905600 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚48232448 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 3897249697084 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 3905421893120 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚41945088 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 125829120 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚262 PCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚59049984 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 5368709120 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚209 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚438146652667904 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚322 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚54217736 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚41945088 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚442043943713008 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 122306526 % _timestamp‚1.7866629414253929e+09 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚429 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚37236149 C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚4289 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚59807411 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚444659204249088 JDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚262 JDeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4289 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚61478912 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 4628521206 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚438146635628544 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚50787696 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚388329882129920 BDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4876 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 _step‚1329 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚54213449 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 845178880 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 122905600 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚223 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 4130134003200 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 4654602240 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚4287 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚440753841405952 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚440753841405952 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚384199796359168 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 831258624 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚322 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚444659263299072 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚444659204249088 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚ 114025147 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚59807409 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚13551547 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 125829120 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚4287 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚440753824301056 KBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚4876 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚50787614 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 5494538240 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚262 IBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚82 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 4552982528 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 125829120Ž/Ã/Ê'ø%ÄO WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚41348020 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚322 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 5368709120 UHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚68876288 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚442043885325628 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 M@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚59049984 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 3905379948032 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚59807411 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚384199817003008 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚20643840 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚322 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚388329951006208 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 5368709120 PCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚58387380 G>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4876 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 _step‚1329 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚54213449 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 845178880 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 122905600 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚223 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 4130134003200 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 4654602240 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚4287 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚440753841405952 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚440753841405952 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚384199796359168 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 831258624 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚322 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚444659263299072 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚444659204249088 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚ 114025147 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚59807409 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚13551547 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 125829120 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚4287 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚440753824301056 KBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚4876 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚50787614 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 5494538240 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚262 IBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚82 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 4552982528 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 125829120 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 3905379948032 D;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚5264 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚37236147 PIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚80 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 4517085184 F>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 122905600 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚48232448 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 3897249697084 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 3905421893120 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚41945088 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 125829120 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚262 PCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚59049984 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 5368709120 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚209 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚438146652667904 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚322 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚54217736 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚41945088 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚442043943713008 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 122306526 % _timestamp‚1.7866629414253929e+09 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚429 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚37236149 C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚4289 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚59807411 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚444659204249088 JDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚262 JDeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4289 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚61478912 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 4628521206 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚438146635628544 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚50787696 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚388329882129920 BDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59053056 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚440780146638848 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚59810964 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 NADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚59053056 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 5368709120 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚322 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚4293 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 4130333226496 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚ 114027698 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 3905611016704 B9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚5264 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚59810962 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 4552982528 NGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚69 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚13552271 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚4295 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚41948160 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 3905611016704 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚50790694 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚13552340 SFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚66776064 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 4517085184 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚59810964 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 114031993 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚442070360079204 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚384222843731968 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 125829120 % _timestamp‚1.7866629415991352e+09 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚444685757655552 EDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521206 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 3905652964864 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚37238354 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚438172838461440 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚322 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚262 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚59810962 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚262 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚440780163743744 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚54221029 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚50790623 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 3897521617764 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚ 114027698 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚4293 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚388353243734528 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 122306526 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚ 114031993 E6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚322 >5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5264 PDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654602240 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚440780146638848 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 122905600 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 D=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚60 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚440780163743744 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚429 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 DDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59053056 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚440780146638848 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚59810964 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 NADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚59053056 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 5368709120 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚322 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚4293 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 4130333226496 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚ 114027698 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 3905611016704 B9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚5264 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚59810962 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 4552982528 NGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚69 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚13552271 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚4295 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚41948160 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 3905611016704 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚50790694 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚13552340 SFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚66776064 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 4517085184 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚59810964 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 114031993 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚442070360079204 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚384222843731968 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 125829120 % _timestamp‚1.7866629415991352e+09 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚444685757655552 EDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521206 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 3905652964864 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚37238354 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚438172838461440 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚322 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚262 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚59810962 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚262 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚440780163743744 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚54221029 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚50790623 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 3897521617764 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚ 114027698 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚4293 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚388353243734528 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 122306526 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚ 114031993 E6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚322 >5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5264 PDevižfÁceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654602240 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚440780146638848 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 122905600 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 D=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚60 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚440780163743744 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚429 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 DfÒ2þ%- _wandbruntime‚4290  _runtime‚4290ä·øvÿ%ð _step‚1331 0loss/FRt2/assignment_loss‚0.3864366961696489 0loss/FRt1/detection_loss‚0.47771682127330045 /loss/FRt2/detection_loss‚0.4681627000353601 &loss/total_loss‚1.8713744230939353 epoch‚40 % _timestamp‚1.7866629416002748e+09 0loss/FRt1/assignment_loss‚0.5390582056156259 trainer/global_step‚16149 _runtime‚4290.130083113ªIÏ{€&õ 0loss/FRt2/assignment_loss‚0.3864366961696489 0loss/FRt1/detection_loss‚0.47771682127330045 /loss/FRt2/detection_loss‚0.4681627000353601 &loss/total_loss‚1.8713744230939353 epoch‚40 % _timestamp‚1.7866629416002748e+09 0loss/FRt1/assignment_loss‚0.5390582056156259 trainer/global_step‚16149 _runtime‚4290.130083113 _step‚1331³ ¶–$ 2&- _wandbruntime‚4302  _runtime‚4302ú#È‚& _runtime‚4302.861147519 _step‚1332 & lr-AdamW/pg1‚0.0001349395933885632 & lr-AdamW/pg2‚0.0001349395933885632 trainer/global_step‚16199 $ _timestamp‚1.786662954331073e+095@̓&Ç & lr-AdamW/pg2‚0.0001349395933885632 trainer/global_step‚16199 $ _timestamp‚1.786662954331073e+09 _runtime‚4302.861147519 _step‚1332 & lr-AdamW/pg1‚0.0001349395933885632´ *Ô2„&- _wandbruntime‚4302  _runtime‚4302´}4˜Å'…&¿O `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚439455942344704 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 3908811507692 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 KDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚60 G>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4876 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚445986966954496 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚443364736813036 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚439455925305344 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 122306526 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚4287 JBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚665 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚4287 KBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚4876 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚13592483 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 4628521206 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚ 114363610 D;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚5264 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚59985061 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 125829120 _step‚1333 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚54382838 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚262 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17104896 JDeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4289 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 4517085184 F?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚60 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚50938806 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚209 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚429 F>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚4289 BDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚445986907904512 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚442069958295552 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚442069941190656 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 4552982528 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 4142425584128 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 3916966713856 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 122905600 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚262 % _timestamp‚1.7866629543321278e+09 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 845178880 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚385352185643008 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚442069958295552 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 5368709120 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 125829120 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚322 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 3917008658944 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚41945088 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚37346399 PIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚74 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚41940992 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚322 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚442069941190656 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚37346397 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚385352164999168 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 831258624 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 5494538240 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 5368709120 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚322 IBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚60 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 3916966713856 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 5368709120 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚262 @7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚5264 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚20643840 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚443364795200416 trainer/global_step‚16199 _runtime‚4302.865899028 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚59985059 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 4654602240 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 125829120 JDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚445986907904512 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚442069958295552 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚442069941190656 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 4552982528 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 4142425584128 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 3916966713856 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 122905600 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚262 % _timestamp‚1.7866629543321278e+09 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 845178880 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚385352185643008 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚442069958295552 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 5368709120 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 125829120 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚322 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 3917008658944 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚41945088 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚37346399 PIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚74 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚41940992 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚322 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚442069941190656 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚37346397 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚385352164999168 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 831258624 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 5494538240 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 5368709120 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚322 IBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚60 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 3916966713856 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 5368709120 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚262 @7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚5264 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚20643840 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚443364795200416 trainer/global_step‚16199 _runtime‚4302.865899028 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚59985059 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 4654602240 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 125829120 JDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4876 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚445986966954496 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚443364736813036 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚439455925305344 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 122306526 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚4287 JBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚665 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚4287 KBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚4876 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚13592483 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 4628521206 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚ 114363610 D;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚5264 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚59985061 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 125829120 _step‚1333 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚54382838 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚262 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17104896 JDeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4289 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 4517085184 F?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚60 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚50938806 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚209 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚429 F>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚4289 BDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59053056 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 5368709120 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚4293 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 4552982528 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚13593211 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚ 114370450 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 831258624 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 122306526 >5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5264 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚446013520364032 H@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚665 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚439482111098880 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 I@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚4876 _runtime‚4303.045970569 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚4295 P6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚322 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚443391153179184 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚ 114370450 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚41348068 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 5494538240 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 3917239730688 NADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚59053056 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚442096280633344 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚442096263528448 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 4142627428864 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚389517839800832 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚385375233015808 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 D=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚60 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 _step‚1334 SFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚66776064 B6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚322 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 122905600 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚442096280633344 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 114374745 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚54386131 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚262 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚41948160 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚443391211566612 NADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚58387428 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 3909042080304 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚262 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚54381838 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚20643840 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚439482128138240 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚322 % _timestamp‚1.7866629545155823e+09 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 5494538240 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚59988612 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚442096263528448 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 4552982528 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚13593288 G@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚60 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚37348604 NGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚77 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 5494538240 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 5368709120 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚54386131 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚41948160 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚389517906576896 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚429 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 3917197782528 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654602240 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521206 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 trainer/global_step‚16199 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚54381838 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚50941892 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚385375212371968 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚322 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚ 114374745 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 4654602240 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚446013461310976 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 122905600 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚46132224 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 3909083428372 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 125829120 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17104896 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 3917197782528 G@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚79 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚209 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.allÕTý].peak‚ 845178880 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 4517085184 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17104896 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚223 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 DDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚50941813 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 5368709120 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚262à·4¬Ô&‰&ÎM VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚37348604 NGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚77 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 5494538240 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 5368709120 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚54386131 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚41948160 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚389517906576896 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚429 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 3917197782528 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654602240 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521206 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 trainer/global_step‚16199 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚54381838 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚50941892 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚385375212371968 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚322 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚ 114374745 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 4654602240 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚446013461310976 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 122905600 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚46132224 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 3909083428372 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 125829120 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17104896 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 3917197782528 G@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚79 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚209 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 845178880 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 4517085184 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17104896 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚223 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 DDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚50941813 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 5368709120 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚262 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 3917239730688 EDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59053056 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 5368709120 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚4293 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 4552982528 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚13593211 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚ 114370450 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 831258624 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 122306526 >5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5264 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚446013520364032 H@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚665 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚439482111098880 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 I@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚4876 _runtime‚4303.045970569 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚4295 P6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚322 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚443391153179184 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚ 114370450 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚41348068 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 5494538240 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 3917239730688 NADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚59053056 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚442096280633344 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚442096263528448 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 4142627428864 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚389517839800832 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚385375233015808 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 D=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚60 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 _step‚1334 SFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚66776064 B6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚322 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 122905600 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚442096280633344 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 114374745 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚54386131 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚262 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚41948160 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚443391211566612 NADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚58387428 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 3909042080304 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚262 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚54381838 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚20643840 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚439482128138240 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚322 % _timestamp‚1.7866629545155823e+09 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 5494538240 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚59988612 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚442096263528448 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 4552982528 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚13593288 G@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚60¶ ‡:ò2Š&- _wandbruntime‚4303  _runtime‚4303ÞF"ˆx‹&ò 0loss/FRt2/assignment_loss‚0.3804121794686011 0loss/FRt1/detection_loss‚0.48315166792789255 0loss/FRt2/detection_loss‚0.47692989867385427 epoch‚40 trainer/global_step‚16199 _runtime‚4303.046475523 &loss/total_loss‚1.8319607449347608 % _timestamp‚1.7866629545166492e+09 _step‚1335 1loss/FRt1/assignment_loss‚0.49146699886441286ô[²}Œ&÷ 0loss/FRt2/detection_loss‚0.47692989867385427 epoch‚40 trainer/global_step‚16199 _runtime‚4303.046475523 &loss/total_loss‚1.8319607449347608 % _timestamp‚1.7866629545166492e+09 _step‚1335 1loss/FRt1/assignment_loss‚0.49146699886441286 0loss/FRt2/assignment_loss‚0.3804121794686011 0loss/FRt1/detection_loss‚0.48315166792789255· 1‰Þ®¡&‚(:– ¬ ùÓ™á«á gpu.0.gpu‚64.0 gpu.0.memory‚12, gpu.0.memoryAllocated‚7.0313262939453125) gpu.0.memoryAllocatedBytes‚ 6039863296 gpu.0.temp‚36.0 gpu.0.powerWatts‚224.424' gpu.0.enforcedPowerLimitWatts‚500.0) gpu.0.powerPercent‚44.884800000000006 gpu.0.smClock‚1410 gpu.0.memoryClock‚1593$ gpu.0.correctedMemoryErrors‚1955# gpu.0.uncorrectedMemoryErrors‚0択Ž&‚(:“ ¬ ùÓ¸èòâ cpu‚27.238642573064745* disk./.usagePercent‚32.702542715983526 disk.dm-1.in‚39549.48828125 disk.drbd2165.in‚0 disk.dm-0.in‚ 44120.3984375 disk.sda3.in‚39550.19921875 disk.sda1.in‚0# proc.memory.rssMB‚ 2299.55078125% disk./.usageGB‚487.29151916503906 disk.sda.in‚39550.19921875 disk.sda1.out‚0 disk.sda2.in‚0* proc.memory.percent‚14.035344123840332 network.sent‚ 2.1079934e+07 network.recv‚ 8.098752e+06 disk.dm-1.out‚ 79.58203125 disk.sda3.out‚ 79.58203125 disk.sda2.out‚0! disk.nvme0n1.in‚ 44120.3984375% disk.nvme0n1.out‚61065.2431640625 proc.cpu.threads‚17 disk.drbd2165.out‚0" disk.dm-0.out‚61065.2431640625# disk.nvme0c0n1.in‚ 44120.3984375' disk.nvme0c0n1.out‚61065.2431640625 disk.sda.out‚ 79.58203125$ memory_percent‚68.27006340026855( proc.memory.availableMB‚ 5198.6328125¯”Æ2&- _wandbruntime‚4315  _runtime‚4315»@CË&Å ' lr-AdamW/pg2‚0.00013151977042625561 trainer/global_step‚16249 % _timestamp‚1.7866629667040856e+09 _runtime‚4315.234134024 _step‚1336 ' lr-AdamW/pg1‚0.00013151977042625561O.‘8Б&Ê _runtime‚4315.234134024 _step‚1336 ' lr-AdamW/pg1‚0.00013151977042625561 ' lr-AdamW/pg2‚0.00013151977042625561 trainer/global_step‚16249 % _timestamp‚1.7866629667040856e+09¸ P¼ã 2’&- _wandbruntime‚4315  _runtime‚4315'ç.BÅ'“&¿O SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 4552982528 PCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚59049984 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 4552982528 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚390659255910912 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚54543653 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚429 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 trainer/global_step‚16249 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚4287 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚ 114710651 JBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚665 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 122306526 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚262 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚262 _runtime‚4315.235443938 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 3928595424768 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 PCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚58387380 IBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚60 F?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚60 % _timestamp‚1.7866629667051065e+09 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 122905600 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 5494538240 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚60162711 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚444685588300444 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚41348020 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 5368709120 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 5368709120 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 125829120 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚447314611559936 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 4654602240 PIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚71 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚386504554283008 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚46135296 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚443386075185152 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚447314611559936 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚443386058080256 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚54547940 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 5494538240 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17104896 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 125829120 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 3928553479680 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚443386058080256 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚51090450 JDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4876 JDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 KBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚4876 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 831258624 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17104896 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚223 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚440765232021504 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚41945088 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 4654602240 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚37456647 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 4154655492608 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚444685646687824 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 _step‚1337 F>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 5494538240 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚41945088 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚51090523 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚13633803 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚20643840 KDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚60 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚447314670609920 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚440765214982144 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚37456649 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 845178880 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4289 UHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚66779136 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 4517085184 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 3920373318300 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚322 C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚4289 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0ìiˆ"”&ÄO IBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚60 F?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚60 % _timestamp‚1.7866629667051065e+09 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 122905600 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 5494538240 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚60162711 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚444685588300444 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚41348020 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 5368709120 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 5368709120 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 125829120 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚447314611559936 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 4654602240 PIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚71 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚386504554283008 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚46135296 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚443386075185152 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚447314611559936 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚443386058080256 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚54547940 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 5494538240 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17104896 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 125829120 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 3928553479680 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚443386058080256 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚51090450 JDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4876 JDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 KBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚4876 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 831258624 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17104896 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚223 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚440765232021504 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚41945088 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 4654602240 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚37456647 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 4154655492608 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚444685646687824 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 _step‚1337 F>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 5494538240 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚41945088 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚51090523 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚13633803 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚20643840 KDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚60 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚447314670609920 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚440765214982144 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚37456649 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 845178880 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4289 UHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚66779136 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 4517085184 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 3920373318300 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚322 C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚4289 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 4552982528 PCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚59049984 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 4552982528 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_by~ÆBtes.all.allocated‚390659255910912 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚54543653 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚429 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 trainer/global_step‚16249 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚4287 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚ 114710651 JBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚665 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 122306526 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚262 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚262 _runtime‚4315.235443938 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 3928595424768 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 PCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚58387380¹ «}‰:2•&- _wandbruntime‚4315  _runtime‚4315AÃÈÝÏ&–&ÉM NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚60166264 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654602240 NGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚69 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 4154948228608 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚444712004666592 IBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚60 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚209 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 B6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚322 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚322 H@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚665 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚262 D=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚60 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚443412397522944 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚447341164966400 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚41948160 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚444712063054020 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 _runtime‚4315.423238041 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚ 114717497 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚443412380418048 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521206 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 5368709120 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚322 % _timestamp‚1.7866629668928175e+09 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚223 G@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚60 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚4293 PDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚60166264 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚386527601655808 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 5494538240 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚54546940 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 3928784548352 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 122905600 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 5494538240 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 4654602240 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚ 114713202 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚447341224019456 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 4552982528 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚20643840 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 _step‚1338 >5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5264 DDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59053056 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 3928784548352 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 122905600 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 3920645238980 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚262 G@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚71 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 3928826496512 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 3928826496512 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 114717497 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚440791417815040 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚41348068 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚60166262 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚37458852 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚386527581011968 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚48229376 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 5368709120 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 5368709120 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 EDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚51093453 NADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚58387428 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚37458854 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚54551233 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚60166262 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 3920603890912 NADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚59053056 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 125829120ÅÂÔ&—&ÎM H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚4293 PDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚60166264 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚386527601655808 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 5494538240 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚54546940 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 3928784548352 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 122905600 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 5494538240 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 4654602240 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚ 114713202 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚447341224019456 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 4552982528 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚20643840 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 _step‚1338 >5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5264 DDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59053056 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 3928784548352 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 122905600 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 3920645238980 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚262 G@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚71 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 3928826496512 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 3928826496512 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 114717497 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚440791417815040 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚41348068 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚60166262 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚37458852 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚386527581011968 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚48229376 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 5368709120 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 5368709120 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 EDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚51093453 NADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚58387428 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚37458854 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚54551233 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚60166262 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 3920603890912 NADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚59053056 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 125829120 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚60166264 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654602240 NGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚69 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 4154948228608 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚444712004666592 IBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚60 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚209 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 B6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚322 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚322 H@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚665 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚262 D=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚60 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚443412397522944 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚447341164966400 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚41948160 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚444712063054020 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 _runtime‚4315.423238041 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚ 114717497 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚443412380418048 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521206 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 5368709120 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚322 % _timestamp‚1.7866629668928175e+09 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚223 G@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚60º )‰ÅD2˜&- _wandbruntime‚4315  _runtime‚4315ùGO’t™&î 0loss/FRt1/assignment_loss‚0.5473922136100615 /loss/FRt1/detection_loss‚0.5188388369233559 /loss/FRt2/detection_loss‚0.4871746425087123 epoch‚40 trainer/global_step‚16249 % _timestamp‚1.7866629668939025e+09 _runtime‚4315.423810722 0loss/FRt2/assignment_loss‚0.4565199929526186 %loss/total_loss‚2.009925685994748 _step‚1339àõéyš&ó 0loss/FRt1/assignment_loss‚0.5473922136100615 /loss/FRt1/detection_loss‚0.5188388369233559 /loss/FRt2/detection_loss‚0.4871746425087123 epoch‚40 trainer/global_step‚16249 % _timestamp‚1.7866629668939025e+09 _runtime‚4315.423810722 0loss/FRt2/assignment_loss‚0.4565199929526186 %loss/total_loss‚2.009925685994748 _step‚1339» ÆåUΖ›&‚(:‹ » ùÓ®›á gpu.0.gpu‚57.0 gpu.0.memory‚10, gpu.0.memoryAllocated‚7.0313262939453125) gpu.0.memoryAllocatedBytes‚ 6039863296 gpu.0.temp‚35.0 gpu.0.powerWatts‚120.806' gpu.0.enforcedPowerLimitWatts‚500.0 gpu.0.powerPercent‚24.1612 gpu.0.smClock‚1410 gpu.0.memoryClock‚1593$ gpu.0.correctedMemoryErrors‚1955# gpu.0.uncorrectedMemoryErrors‚0Ä¢œ&‚(:— » ùÓ®¯¦ä& disk.nvme0n1.out‚61069.70947265625 disk.dm-1.out‚ 79.76171875 disk.dm-0.in‚44121.54296875$ memory_percent‚68.26889514923096# proc.memory.rssMB‚ 2299.55078125 disk.sda2.in‚0 network.sent‚ 2.1154774e+07$ disk.nvme0c0n1.in‚44121.54296875 disk.sda1.out‚0 disk.dm-1.in‚40680.58203125# disk.dm-0.out‚61069.70947265625) proc.memory.availableMB‚ 5198.82421875 cpu‚27.23814725848266 disk.sda3.in‚40680.58203125 disk.sda3.out‚ 79.76171875 disk.drbd2165.in‚0 disk.drbd2165.out‚0 proc.cpu.threads‚17 disk.sda2.out‚0 network.recv‚ 8.120344e+06$ disk./.usageGB‚487.2914924621582( disk.nvme0c0n1.out‚61069.70947265625 disk.sda.in‚40680.58203125 disk.sda.out‚ 79.76171875" disk.nvme0n1.in‚44121.54296875* proc.memory.percent‚14.035344123840332) disk./.usagePercent‚32.70254092393078 disk.sda1.in‚0…»½2&- _wandbruntime‚4328  _runtime‚4328PWn5Ëž&Å _runtime‚4328.017479062 _step‚1340 ' lr-AdamW/pg1‚0.00012813967129336864 ' lr-AdamW/pg2‚0.00012813967129336864 trainer/global_step‚16299 % _timestamp‚1.7866629794874678e+09LìºÌП&Ê % _timestamp‚1.7866629794874678e+09 _runtime‚4328.017479062 _step‚1340 ' lr-AdamW/pg1‚0.00012813967129336864 ' lr-AdamW/pg2‚0.00012813967129336864 trainer/global_step‚16299¼ Á-@2 &- _wandbruntime‚4328  _runtime‚4328¬o®1"¡&¾O TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17104896 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚446006498175232 JBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚665 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚61478912 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 3940140245504 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4289 IBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚68 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 3931935128908 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚223 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 122306526 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 5368709120 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚51241315 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 4166977053696 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 125829120 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 3940182190592 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚448642374265344 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚209 UHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚60487680 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 4166937209856 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 M@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚59049984 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 831258624 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚322 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚448642315215360 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚60340359 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚51241383 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 5494538240 F>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚54713042 PCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚58387380 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 KDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚60 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚448642374265344 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚41945088 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 4628521206 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 122905600 JDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4876 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 4654602240 PCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚59049984 C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚4289 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17104896 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 845178880 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚387656902279168 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚262 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚444702192074752 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚60340361 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚4287 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚54708755 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚ 115053403 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚60340361 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚37566897 BDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚448642315215360 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚37566899 F8DeviceStatsMonitor.on_train_batch_star›¨ ¡“t/active.all.freed‚ 115049114 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚322 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚262 _step‚1341 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚60340359 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚54708755 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 4552982528 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚20643840 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚322 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚444702192074752 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚41945088 KBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚4876 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚39843840 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚322 JDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚54713042 PCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚58387380 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 KDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚60 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚448642374265344 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚41945088 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 4628521206 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 122905600 JDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4876 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 4654602240 PCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚59049984 C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚4289 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17104896 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 845178880 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚387656902279168 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚262 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚444702192074752 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚60340361 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚4287 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚54708755 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚ 115053403 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚60340361 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚37566897 BDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚448642315215360 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚37566899 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚ 115049114 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚322 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚262 _step‚1341 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚60340359 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚54708755 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 4552982528 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚20643840 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚322 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚444702192074752 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚41945088 KBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚4876 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚39843840 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚322 JDeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4289 IBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚68 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 3931935128908 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚223 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 122306526 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 5368709120 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚51241315 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0½ µZÙE2£&- _wandbruntime‚4328  _runtime‚4328ä…£ Ï&¤&ÉM I@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚4876 D=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚60 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚51244391 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 831258624 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 G@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚60 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 122905600 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚322 NADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚58387428 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521206 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚442100707491840 trainer/global_step‚16299 K>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59053056 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 3932165701520 H@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚665 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 122905600 _runtime‚4328.182487423 D6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚322 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚13675287 _step‚1342 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚41948160 G@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚65 SFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚66776064 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚448668868621824 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚209 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚223 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 122306526 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 3940371314176 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚13675224 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚391847129074688 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 4167179422720 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 125829120 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚41948160 NADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚59053056 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚ 115060249 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚4293 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚444728514412544 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚4295 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚448668927674880 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚387679949651968 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚444728497307648 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 3940413262336 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 4167225554944 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚54716335 PDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 115060249 E8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚262 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17104896 E5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5264 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17104896 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚444728497307648 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚54712042 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚37569102 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚41348068 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚ 115055954 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚60343914 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 5368709120 IBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚60 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 4552982528 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 BDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654602240 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚51244326 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 125829120 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 125829120 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚262 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 5368709120 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 3940371314176 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚444728514412544 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚60343912Ó`Ÿ'Ô&¥&ÎM H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚4293 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚444728514412544 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚4295 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚448668927674880 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚387679949651968 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚444728497307648 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 3940413262336 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 4167225554944 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚54716335 PDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 115060249 E8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚262 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17104896 E5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5264 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17104896 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚444728497307648 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚54712042 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚37569102 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚41348068 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚ 115055954 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚60343914 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 5368709120 IBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚60 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 4552982528 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 BDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654602240 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚51244326 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 125829120 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 125829120 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚262 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 5368709120 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 3940371314176 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚444728514412544 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚60343912 I@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚4876 D=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚60 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚51244391 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 831258624 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 G@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚60 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 122905600 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚322 NADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚58387428 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521206 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚442100707491840 trainer/global_step‚16299 K>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59053056 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 3932165701520 H@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚665 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 122905600 _runtime‚4328.182487423 D6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚322 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚13675287 _step‚1342 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚41948160 G@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚65 SFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚66776064 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚448668868621824 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚209 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚223 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 122306526 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 3940371314176 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚13675224 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚391847129074688 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 4167179422720 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 125829120 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚41948160 NADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚59053056 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚ 115060249¾ )Ãrš2¦&- _wandbruntime‚4328  _runtime‚4328 jçu§&ï 1loss/FRt1/assignment_loss‚0.47450976873866574 /loss/FRt2/assignment_loss‚0.408239499401053 /loss/FRt1/detection_loss‚0.4884622079618498 /loss/FRt2/detection_loss‚0.4688160113790942 epoch‚40 % _timestamp‚1.7866629796531706e+09 _step‚1343 &loss/total_loss‚1.8400274874806628 trainer/global_step‚16299 _runtime‚4328.1830148668ϵJz¨&ô 1loss/FRt1/assignment_loss‚0.47450976873866574 /loss/FRt2/assignment_loss‚0.408239499401053 /loss/FRt1/detection_loss‚0.4884622079618498 /loss/FRt2/detection_loss‚0.4688160113790942 epoch‚40 % _timestamp‚1.7866629796531706e+09 _step‚1343 &loss/total_loss‚1.8400274874806628 trainer/global_step‚16299 _runtime‚4328.183014866¿ ~¥’ºo©&‚(:– Ê ùÓᕉá gpu.0.gpu‚64.0 gpu.0.memory‚12, gpu.0.memoryAllocated‚7.0313262939453125) gpu.0.memoryAllocatedBytes‚ 6039863296 gpu.0.temp‚37.0 gpu.0.powerWatts‚197.498' gpu.0.enforcedPowerLimitWatts‚500.0) gpu.0.powerPercent‚39.499599999999994 gpu.0.smClock‚1410 gpu.0.memoryClock‚1593$ gpu.0.correctedMemoryóL2Errors‚1955# gpu.0.uncorrectedMemoryErrors‚0k<[”ª&‚(:‰ Ê ùÓ‹úúâ disk.sda3.in‚40680.58203125$ memory_percent‚68.26896667480469% proc.memory.availableMB‚ 5198.8125* proc.memory.percent‚14.035344123840332 proc.cpu.threads‚17% disk./.usageGB‚487.29136657714844 disk.sda.out‚ 80.1953125 disk.sda2.out‚0 disk.nvme0n1.in‚ 44121.546875 disk.dm-1.in‚40680.58203125 disk.drbd2165.in‚0# proc.memory.rssMB‚ 2299.55078125 network.sent‚ 2.1229628e+07" disk.nvme0c0n1.in‚ 44121.546875 disk.sda1.in‚0" disk.dm-0.out‚61113.9208984375 cpu‚27.237804791632957 network.recv‚ 8.152041e+06' disk.nvme0c0n1.out‚61113.9208984375 disk.sda.in‚40680.58203125 disk.sda1.out‚0% disk.nvme0n1.out‚61113.9208984375 disk.drbd2165.out‚0 disk.dm-0.in‚ 44121.546875 disk.sda3.out‚ 80.1953125) disk./.usagePercent‚32.70253247568215 disk.sda2.in‚0 disk.dm-1.out‚ 80.1953125kËY×2«&- _wandbruntime‚4340  _runtime‚4340^Òɬ&à ' lr-AdamW/pg2‚0.00012479951308735405 trainer/global_step‚16349 # _timestamp‚1.78666299200344e+09 _runtime‚4340.533430548 _step‚1344 ' lr-AdamW/pg1‚0.00012479951308735405'§®Î­&È ' lr-AdamW/pg1‚0.00012479951308735405 ' lr-AdamW/pg2‚0.00012479951308735405 trainer/global_step‚16349 # _timestamp‚1.78666299200344e+09 _runtime‚4340.533430548 _step‚1344À ÷Rò2®&- _wandbruntime‚4340  _runtime‚43400ú!Ä'¯&¾O PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚60518011 PCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚59049984 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚20643840 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚46135296 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚449970077920768 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 4654602240 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4289 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚447327291275260 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17104896 IBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚67 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚13715278 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 4517085184 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 4552982528 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚54878144 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 F?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚60 _runtime‚4340.534590269 UHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚66779136 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚388809291563008 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 M@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚59049984 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 122905600 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚392988560200704 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 122306526 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚37677147 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚449970018870784 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚13715343 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚447327349662640 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 5494538240 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚41348020 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 trainer/global_step‚16349 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚54873857 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚60518009 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚449970018870784 PCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚58387380 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 5368709120 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 125829120 C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚4289 G>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4876 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚41945088 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 3951727011328 PIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚65 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚54878144 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 4654602240 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚60518009 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚51392492 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 4628521206 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚223 KBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚4876 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 $ _timestamp‚1.786662992004354e+09 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 3951727011328 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚ 115396155 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚392988493421568 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 5368709120 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚262 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚41945088 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 5494538240 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚322 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 4179222502400 IBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚60 F>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 JBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚665 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚262 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚446018308964352 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚449970077920768 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 4552982528 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17104896 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 122905600 D;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚5264 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚209 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚443383794335744 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 3951768956416 JDeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4289 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚447327291275260 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17104896 IBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚67 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚13715278 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 4517085184 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 4552982528 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚54878144 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 F?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚60 _runtime‚4340.534590269 UHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚66779136 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚388809291563008 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 M@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚59049984 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 122905600 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚392988560200704 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 122306526 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚37677147 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚449970018870784 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚13715343 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚447327349662640 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 5494538240 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚41348020 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 trainer/global_step‚16349 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚54873857 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚60518009 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚449970018870784 PCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚58387380 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 5368709120 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 125829120 C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚4289 G>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4876 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚41945088 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 3951727011328 PIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚65 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚54878144 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 4654602240 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚60518009 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚51392492 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 4628521206 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚223 KBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚4876 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 $ _timestamp‚1.786662992004354e+09 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 3951727011328 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚ 115396155 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚392988493421568 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 5368709120 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚262 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚41945088 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 5494538240 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚322 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 4179222502400 IBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚60 F>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 JBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚665 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚262 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚446018308964352 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚449970077920768 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 4552982528 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17104896 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 122905600 D;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚5264 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚209 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚443383794335744 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 3951768956416 J8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚262 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 122905600 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17104896 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚262 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 3952000028160 NADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚59053056 G@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚61 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 B6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚322 K>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59053056 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚60521562 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚37679354 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚37679352 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 831258624 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 5494538240 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 125829120 % _timestamp‚1.7866629921677074e+09 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚ 115398706 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚446044614197248 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 3951958080000 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚13716054 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚ 115403001 >5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5264 PDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654602240 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚41948160 EDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚262 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 3951958080000 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚388832338935808 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚20643840 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚443409997168640 trainer/global_step‚16349 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚51395467 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚4293 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚446044631302144 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 122905600 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 4552982528 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚51395406 NADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚58387428 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚449996572277248 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚447353707641408 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚322 IBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚60 D=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚60 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚4295 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚449996631330304 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 3943768860196 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚322 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚39840768 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 122306526 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚60521562 EDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521206 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 5494538240 G@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚60 _step‚1346 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 4654602240 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 125829120 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚446044614197248 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 4552982528 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 5494538240 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 5368709120 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 5368709120 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚209 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚54881437 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17104896 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚41948160 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚13716113 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 845178880 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 4179471563776 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚322 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 115403001 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0¾¶.d³&ÎM REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17104896 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚41948160 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚13716113 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 845178880 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 4179471563776 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚322 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 115403001 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚60521564 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peakË{¨p#‚262 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 122905600 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17104896 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚262 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 3952000028160 NADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚59053056 G@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚61 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 B6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚322 K>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59053056 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚60521562 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚37679354 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚37679352 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 831258624 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 5494538240 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 125829120 % _timestamp‚1.7866629921677074e+09 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚ 115398706 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚446044614197248 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 3951958080000 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚13716054 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚ 115403001 >5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5264 PDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654602240 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚41948160 EDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚262 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 3951958080000 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚388832338935808 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚20643840 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚443409997168640 trainer/global_step‚16349 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚51395467 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚4293 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚446044631302144 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 122905600 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 4552982528 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚51395406 NADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚58387428 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚449996572277248 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚447353707641408 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚322 IBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚60 D=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚60 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚4295 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚449996631330304 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 3943768860196 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚322 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚39840768 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 122306526 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚60521562 EDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521206 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 5494538240 G@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚60 _step‚1346 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 4654602240 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 125829120 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚446044614197248 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 4552982528 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 5494538240 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 5368709120 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 5368709120 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚209 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚54881437 ó»kô2´&- _wandbruntime‚4340  _runtime‚4340]=Þluµ&ï /loss/FRt1/detection_loss‚0.4857174534395331 /loss/FRt2/detection_loss‚0.4881901960380111 epoch‚40 trainer/global_step‚16349 _step‚1347 0loss/FRt2/assignment_loss‚0.3813902411991899 &loss/total_loss‚1.8660495342836132 % _timestamp‚1.7866629921686592e+09 _runtime‚4340.698452186 0loss/FRt1/assignment_loss‚0.5107516436068792~¼ðz¶&ô _runtime‚4340.698452186 0loss/FRt1/assignment_loss‚0.5107516436068792 /loss/FRt1/detection_loss‚0.4857174534395331 /loss/FRt2/detection_loss‚0.4881901960380111 epoch‚40 trainer/global_step‚16349 _step‚1347 0loss/FRt2/assignment_loss‚0.3813902411991899 &loss/total_loss‚1.8660495342836132 % _timestamp‚1.7866629921686592e+09à µ>h–·&‚(:‹ Ù ùÓ•á gpu.0.gpu‚59.0 gpu.0.memory‚10, gpu.0.memoryAllocated‚7.0313262939453125) gpu.0.memoryAllocatedBytes‚ 6039863296 gpu.0.temp‚34.0 gpu.0.powerWatts‚120.518' gpu.0.enforcedPowerLimitWatts‚500.0 gpu.0.powerPercent‚24.1036 gpu.0.smClock‚1410 gpu.0.memoryClock‚1593$ gpu.0.correctedMemoryErrors‚1955# gpu.0.uncorrectedMemoryErrors‚0n.涛¸&‚(: Ù ùÓþ¼•ä disk.nvme0n1.in‚ 44121.546875 disk.drbd2165.out‚0 disk.sda3.out‚ 80.5390625* proc.memory.percent‚14.035344123840332( disk.nvme0c0n1.out‚61121.24560546875 disk.sda1.out‚0 disk.sda2.out‚0 disk.dm-1.in‚40680.58203125 disk.dm-1.out‚ 80.5390625 disk.dm-0.in‚ 44121.546875 cpu‚27.237489135653032 network.sent‚ 2.1304649e+07" disk.nvme0c0n1.in‚ 44121.546875 disk.sda.out‚ 80.5390625# disk.dm-0.out‚61121.24560546875) proc.memory.availableMB‚ 5198.49609375 proc.cpu.threads‚17* disk./.usagePercent‚32.710964595837424$ disk./.usageGB‚487.4170112609863 disk.sda2.in‚0& disk.nvme0n1.out‚61121.24560546875 disk.drbd2165.in‚0 disk.sda3.in‚40680.58203125$ memory_percent‚68.27089786529541# proc.memory.rssMB‚ 2299.55078125 network.recv‚ 8.184693e+06 disk.sda.in‚40680.58203125 disk.sda1.in‚0ÞáF:2¹&- _wandbruntime‚4353  _runtime‚4353˜VÊ&˺&Å ' lr-AdamW/pg2‚0.00012149951034033146 trainer/global_step‚16399 % _timestamp‚1.7866630047505348e+09 _runtime‚4353.280563868 _step‚1348 ' lr-AdamW/pg1‚0.00012149951034033146BW:‰Ð»&Ê ' lr-AdamW/pg2‚0.00012149951034033146 trainer/global_step‚16399 % _timestamp‚1.7866630047505348e+09 _runtime‚4353.280563868 _step‚1348 ' lr-AdamW/pg1‚0.00012149951034033146Ä Bxíå2¼&- _wandbruntime‚4353  _runtime‚4353*‚ÒÄ'½&¾O M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚51543268 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚429 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚13755934 UHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚68876288 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚389961639559168 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17104896 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚41348020 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚451297722526208 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚447334408749056 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4289 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚61478912 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚55038959 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚389961660203008 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚448648142762668 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 @7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚5264 PCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚59049984 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚447334425853952 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 4191514241536 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 125829120 F>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚322 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚444693084012544 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚37787399 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 4191562473984 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 122306526 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 5494538240 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚55043246 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17104896 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚447334425853952 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 4552982528 JBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚665 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚55038959 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚444693101051904 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 4552982528 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 831258624 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 5494538240 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚322 M@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚59049984 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 KBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚4876 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 125829120 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚451297722526208 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚447334408749056 D;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚5264 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 5368709120 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚262 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 4654602240 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚41945088 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 845178880 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 125829120 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚ 115734618 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 122905600 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 4654602240 JDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4876 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 trainer/global_step‚16399 _step‚1349 JDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚448648201150048 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚322 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚4287 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚262 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚451297781576192—@1ÚÉ'¾&ÃO UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 122905600 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚322 _runtime‚4353.284896047 IBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚65 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 KDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚60 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚448648201150048 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚322 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚4287 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚262 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚451297781576192 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚51543268 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚429 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚13755934 UHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚68876288 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚389961639559168 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17104896 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚41348020 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚451297722526208 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚447334408749056 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4289 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚61478912 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚55038959 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚389961660203008 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚448648142762668 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 @7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚5264 PCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚59049984 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚447334425853952 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 4191514241536 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 125829120 F>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚322 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚444693084012544 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚37787399 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 4191562473984 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 122306526 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 5494538240 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚55043246 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17104896 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚447334425853952 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 4552982528 JBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚665 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚55038959 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚444693101051904 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 4552982528 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 831258624 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 5494538240 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚322 M@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚59049984 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 KBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚4876 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 125829120 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚451297722526208 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚447334408749056 D;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚5264 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 5368709120 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚262 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 4654602240 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚41945088 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 845178880 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 125829120 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚ 115734618 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 122905600 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 4654602240 JDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4876 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 trainer/global_step‚16399 _step‚1349 JDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚4293 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 4654602240 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 EDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59053056 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚262 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚4295 >5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5264 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 BDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚51546264 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚37789602 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 831258624 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 125829120 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚60699212 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚50326528 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚61478912 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521206 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 4517085184 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 I@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚4876 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 122306526 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 5368709120 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17104896 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚447360731086848 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚209 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚389984707575808 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 5494538240 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚262 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 4191809598976 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚322 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚444719269806080 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 5368709120 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚60699214 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 3963544845824 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚444719286845440 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 5368709120 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 125829120 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚451324275932672 G@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚73 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 115745753 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚448674559128816 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 5494538240 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 G@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚60 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚51546337 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚262 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚41948160 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17104896 NGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚71 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚394176517174784 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 % _timestamp‚1.7866630049215958e+09 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654602240 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚447360731086848 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚429 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 3955330670804 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚60699212 H@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚665 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚13756733 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 _runtime‚ 4353.45223218 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚37789604 _step‚1350 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚ 115741458 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚447360748191744 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 3963544845824 SFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚70970368 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 845178880 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚20643840 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 NADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚59053056 E6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚322 PDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚4293 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 4654602240 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 EDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59053056 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚262 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚4295 >5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5264 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 BDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚51546264 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚37789602 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 831258624 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 125829120 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚60699212 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚50326528 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚61478912 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521206 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 4517085184 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 I@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚4876 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 122306526 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 5368709120 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17104896 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚447360731086848 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚209 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚389984707575808 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 5494538240 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚262 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 4191809598976 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚322 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚444719269806080 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 5368709120 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚60699214 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 3963544845824 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚444719286845440 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 5368709120 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 125829120 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚451324275932672 G@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚73 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 115745753 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚448674559128816 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 5494538240 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 G@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚60 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚51546337 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚262 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚41948160 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17104896 NGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚71 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚394176517174784 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 % _timestamp‚1.7866630049215958e+09 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654602240 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚447360731086848 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚429 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 3955330670804 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚60699212 H@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚665 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚13756733 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 _runtime‚ 4353.45223218 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚37789604 _step‚1350 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚ 115741458 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚447360748191744 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 3963544845824 SFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚70970368 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 845178880 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚20643840 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 NADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚59053056 E6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚322 PDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17104896 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚395621252331520 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 845178880 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 122306526 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 5494538240 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚41945088 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚13805666 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 IBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚60 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚453020490170368 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 122905600 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚446383290220544 PCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚59049984 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 4207953180672 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 4628521206 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚446383273181184 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚322 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 4552982528 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚449041479958528 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 JDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4876 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚209 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 PCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚58387380 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 5368709120 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚453020431120384 JBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚665 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚322 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚55226248 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚453020490170368 D;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚5264 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 5494538240 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚60933917 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚429 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 4207911239680 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 5494538240 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 125829120 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚322 M@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚59049984 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚20643840 BDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 3978968266752 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚60933917 QDDeviceStÅ–n&æatsMonitor.on_train_batch_start/inactive_split.all.allocated‚51743990 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚41348020 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 5368709120 trainer/global_step‚16449 C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚4289 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚37938261 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 3970727880853 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚322 _runtime‚4371.288865817 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 4552982528 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 % _timestamp‚1.7866630227557204e+09 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17104896 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚60933915 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 3978968266752 JDeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4289 KBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚4876 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚453020431120384 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚51743927 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚262 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚55230535È8H­Ê'Ï&ÄO PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚60933917 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚429 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 4207911239680 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 5494538240 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 125829120 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚322 M@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚59049984 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚20643840 BDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 3978968266752 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚60933917 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚51743990 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚41348020 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 5368709120 trainer/global_step‚16449 C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚4289 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚37938261 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 3970727880853 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚322 _runtime‚4371.288865817 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 4552982528 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 % _timestamp‚1.7866630227557204e+09 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17104896 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚60933915 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 3978968266752 JDeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4289 KBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚4876 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚453020431120384 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚51743927 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚262 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚55230535 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 4517085184 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 3970686532833 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚41940992 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17104896 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚395621252331520 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 845178880 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 122306526 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 5494538240 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚41945088 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚13805666 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 IBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚60 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚453020490170368 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 122905600 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚446383290220544 PCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚59049984 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 4207953180672 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 4628521206 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚446383273181184 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚322 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 4552982528 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚449041479958528 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 JDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4876 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚209 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 PCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚58387380 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 5368709120 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚453020431120384 JBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚665 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚322 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚55226248 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚453020490170368 D;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚5264 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 5494538240 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0Ê ©2Ð&- _wandbruntime‚4371  _runtime‚4371‰:³Ï&Ñ&ÉM NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚60937470 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚55233828 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚429 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚37940466 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚450380434467593 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚262 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 125829120 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚41948160 EDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654602240 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 4552982528 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 3979199335424 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 D=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚60 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 3970917105445 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚55229535 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚51746995 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 831258624 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚322 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚60937468 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 4654602240 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚55229535 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚453047043579904 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚51746923 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 5494538240 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 5368709120 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚4295 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 3979241283584 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚60937470 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚4293 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17104896 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚449067802296320 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 3979241283584 NGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚70 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚13806457 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚209 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 P6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚322 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚55233828 G@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚72 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 3979199335424 SFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚66776064 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 5494538240 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 5368709120 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 4208200829952 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚61478912 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 5368709120 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 122905600 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚223 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 3970958453513 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚ 116167003 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚391436325879808 NADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚58387428 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚262 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚4293 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚13806527 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚322 E5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5264 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚446409476014080 IBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚60 K>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59053056 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 122905600 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 116171298 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚391436346523648 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚41348068 G@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚60 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚322 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521206 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 125829120 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 B9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚5264 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 845178880 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 5494538240 H@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚665 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚395644547353600 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 122306526 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17104896 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚449067785191424 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚450380376080165 trainer/global_step‚16449 _step‚1355 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚453047043579904 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚ 116167003 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 125829120 DDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 P6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚322 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚55233828 G@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚72 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 3979199335424 SFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚66776064 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 5494538240 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 5368709120 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 4208200829952 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚61478912 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 5368709120 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 122905600 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚223 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 3970958453513 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚ 116167003 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚391436325879808 NADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚58387428 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚262 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚4293 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚13806527 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚322 E5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5264 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚446409476014080 IBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚60 K>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59053056 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 122905600 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 116171298 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚391436346523648 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚41348068 G@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚60 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚322 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521206 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 125829120 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 B9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚5264 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 845178880 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 5494538240 H@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚665 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚395644547353600 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 122306526 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17104896 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚449067785191424 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚450380376080165 trainer/global_step‚16449 _step‚1355 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚453047043579904 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚ 116167003 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 125829120 DDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654602240 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 4552982528 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 3979199335424 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 D=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚60 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 3970917105445 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚55229535 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚51746995 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 831258624 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚322 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚60937468 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 4654602240 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚55229535 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚453047043579904 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚51746923 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 5494538240 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 5368709120 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚4295 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 3979241283584 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚60937470 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚4293 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0Ë te0I2Ó&- _wandbruntime‚4371  _runtime‚4371|ð’gxÔ&ò trainer/global_step‚16449 % _timestamp‚1.7866630229220622e+09 _runtime‚4371.451904143 _step‚1356 0loss/FRt1/assignment_loss‚0.4233690965635373 1loss/FRt2/assignment_loss‚0.35932586738206507 0loss/FRt1/detection_loss‚0.44838399511115745 0loss/FRt2/detection_loss‚0.46988585103057573 &loss/total_loss‚1.7009648100873356 epoch‚41ùu!™Õ&÷ epoch‚41 trainer/global_step‚16449 % _timestamp‚1.7866630229220622e+09 _runtime‚4371.451904143 _step‚1356 0loss/FRt1/assir«ägnment_loss‚0.4233690965635373 1loss/FRt2/assignment_loss‚0.35932586738206507 0loss/FRt1/detection_loss‚0.44838399511115745 0loss/FRt2/detection_loss‚0.46988585103057573 &loss/total_loss‚1.7009648100873356Ì ~ H“Ö&‚(:ˆ ÷ ùÓÙï”â gpu.0.gpu‚32.0 gpu.0.memory‚5, gpu.0.memoryAllocated‚7.0313262939453125) gpu.0.memoryAllocatedBytes‚ 6039863296 gpu.0.temp‚37.0 gpu.0.powerWatts‚220.06' gpu.0.enforcedPowerLimitWatts‚500.0 gpu.0.powerPercent‚44.012 gpu.0.smClock‚1410 gpu.0.memoryClock‚1593$ gpu.0.correctedMemoryErrors‚1955# gpu.0.uncorrectedMemoryErrors‚0j|s×&‚(:’ ÷ ùÓ»è®ä disk.sda2.out‚0 disk.dm-1.in‚40680.58203125 disk.drbd2165.out‚0 network.recv‚ 8.234245e+06* disk./.usagePercent‚32.710912882315505 disk.sda.out‚ 81.2109375" disk.nvme0n1.in‚44159.56640625% disk.nvme0n1.out‚61173.9404296875 disk.dm-1.out‚ 81.2109375 disk.dm-0.in‚44159.56640625$ memory_percent‚68.60578060150146' disk.nvme0c0n1.out‚61173.9404296875 disk.drbd2165.in‚0" disk.dm-0.out‚61173.9404296875 cpu‚27.243570837670802 disk.sda.in‚40680.58203125 disk.sda1.in‚0 disk.sda1.out‚0 disk.sda2.in‚0) proc.memory.availableMB‚ 5143.62890625# proc.memory.rssMB‚ 2354.24609375) proc.memory.percent‚14.36917781829834 proc.cpu.threads‚17 network.sent‚ 2.1441675e+07$ disk./.usageGB‚487.4162406921387 disk.sda3.in‚40680.58203125 disk.sda3.out‚ 81.2109375$ disk.nvme0c0n1.in‚44159.56640625oÈ#22Ø&- _wandbruntime‚4384  _runtime‚4384×!YÊÙ&Ä _runtime‚ 4384.10145295 _step‚1357 ' lr-AdamW/pg1‚0.00011502081644257801 ' lr-AdamW/pg2‚0.00011502081644257801 trainer/global_step‚16499 % _timestamp‚1.7866630355714195e+09£ÂkAÏÚ&É _step‚1357 ' lr-AdamW/pg1‚0.00011502081644257801 ' lr-AdamW/pg2‚0.00011502081644257801 trainer/global_step‚16499 % _timestamp‚1.7866630355714195e+09 _runtime‚ 4384.10145295Í ¿ºx2Û&- _wandbruntime‚4384  _runtime‚4384‚½ý†Å'Ü&¿O M@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚59049984 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚55391350 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚4287 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚20643840 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 5494538240 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚450357596848128 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚447692562857984 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚454348193825792 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 125829120 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 125829120 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 3990555032576 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚38048513 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 4220243958272 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 122306526 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 4654602240 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 3990596977664 KBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚4876 PIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚77 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚451674811201425 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 5368709120 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚450357596848128 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚450357579743232 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚51895430 BDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚454348134775808 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚61111567 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 IBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚60 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17104896 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚41945088 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4289 D;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚5264 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 845178880 KDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚60 C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚4289 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚ 116502915 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚223 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚13846840 G>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4876 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚ 116507204 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚396785847067136 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 4517085184 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 5494538240 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 JBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚665 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚451674869588805 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚322 F>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 3990596977664 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚51895351 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚13846917 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 4628521206 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 5494538240A™=%Ê'Ý&ÄO HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚322 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 122905600 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 F?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚60 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 4654602240 PCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚59049984 JDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚454348134775808 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚61111567 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 IBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚60 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17104896 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚41945088 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4289 D;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚5264 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 845178880 KDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚60 C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚4289 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚ 116502915 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚223 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚13846840 G>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4876 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚ 116507204 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚396785847067136 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 4517085184 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 5494538240 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 JBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚665 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚451674869588805 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚322 F>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 3990596977664 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚51895351 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚13846917 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 4628521206 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 5494538240 M@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚59049984 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚55391350 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚4287 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚20643840 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 5494538240 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚450357596848128 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚447692562857984 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚454348193825792 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 125829120 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 125829120 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 3990555032576 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚38048513 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 4220243958272 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 122306526 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 4654602240 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 3990596977664 KBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚4876 PIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚77 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚451674811201425 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 5368709120 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚450357596848128 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚450357579743232 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚51895430 BDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 trainer/global_step‚16499 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚454374688182272 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 845178880 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚262 D=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚60 DDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 116514050 _step‚1359 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚ 116514050 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚454374747235328 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚209 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521206 E5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5264 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚41948160 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 122905600 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚447718765690880 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚ 116509755 K>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59053056 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚55394637 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚51898354 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 NADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚59053056 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 5494538240 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 3990828049408 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17104896 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚ 116509755 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 831258624 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 4220445803008 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 NADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚58387428 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚447718748651520 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚55398930 P6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚322 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚4293 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚429 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚262 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚55394637 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚450383919185920 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚55398930 G@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚79 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 4517085184 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚20643840 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 3982478916053 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17104896 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 125829120 EDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654602240 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚450383919185920 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 3990828049408 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 5368709120 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚61115118 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚13847715 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 3990786101248 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚13847638 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚322 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚322 G@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚60 B9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚5264 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚322 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 4552982528 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 % _timestamp‚1.7866630357537694e+09 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 125829120 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚450383902081024 NGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚77 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 3982520264121ZÞ à&ÎM PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 122905600 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 ^JDeviceStatsMonitor.on_train_batch_end/requÅ5kÉ%ested_bytes.large_pool.allocated‚447718765690880 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚ 116509755 K>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59053056 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚55394637 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚51898354 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 NADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚59053056 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 5494538240 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 3990828049408 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17104896 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚ 116509755 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 831258624 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 4220445803008 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 NADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚58387428 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚447718748651520 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚55398930 P6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚322 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚4293 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚429 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚262 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚55394637 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚450383919185920 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚55398930 G@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚79 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 4517085184 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚20643840 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 3982478916053 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17104896 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 125829120 EDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654602240 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚450383919185920 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 3990828049408 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 5368709120 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚61115118 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚13847715 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 3990786101248 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚13847638 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚322 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚322 G@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚60 B9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚5264 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚322 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 4552982528 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 % _timestamp‚1.7866630357537694e+09 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 125829120 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚450383902081024 NGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚77 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 3982520264121 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 trainer/global_step‚16499 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚454374688182272 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 845178880 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚262 D=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚60 DDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 116514050 _step‚1359 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚ 116514050 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚454374747235328 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚209 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521206 E5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5264 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚41948160Ï ¤è-n2á&- _wandbruntime‚4384  _runtime‚4384FÇÝãwâ&ñ 0loss/FRt1/detection_loss‚0.48894906149689155 .loss/FRt2/detection_loss‚0.458360457729659 &loss/total_loss‚1.7842085319032777 % _timestamp‚1.7866630357550182e+09 _step‚1360 1loss/FRt1/assignment_loss‚0.45253940253898156 1loss/FRt2/assignment_loss‚0.38435961013774556 epoch‚41 trainer/global_step‚16499 _runtime‚4384.284898952±žå¯|ã&ö 0loss/FRt1/detection_loss‚0.48894906149689155 .loss/FRt2/detection_loss‚0.458360457729659 &loss/total_loss‚1.7842085319032777 % _timestamp‚1.7866630357550182e+09 _step‚1360 1loss/FRt1/assignment_loss‚0.45253940253898156 1loss/FRt2/assignment_loss‚0.38435961013774556 epoch‚41 trainer/global_step‚16499 _runtime‚4384.284898952Ð >ü!& ä&‚(:• †¡ùÓȃŽá gpu.0.gpu‚10.0 gpu.0.memory‚1, gpu.0.memoryAllocated‚7.0313262939453125) gpu.0.memoryAllocatedBytes‚ 6039863296 gpu.0.temp‚36.0 gpu.0.powerWatts‚137.639' gpu.0.enforcedPowerLimitWatts‚500.0) gpu.0.powerPercent‚27.527800000000003 gpu.0.smClock‚1410 gpu.0.memoryClock‚1593$ gpu.0.correctedMemoryErrors‚1955# gpu.0.uncorrectedMemoryErrors‚0ð åÜ¢å&‚(:— †¡ùÓÔÃã$ disk./.usageGB‚487.4133720397949$ disk.nvme0c0n1.in‚44159.56640625 disk.sda1.in‚0 disk.sda1.out‚0 disk.sda2.in‚0 disk.dm-1.in‚40680.58203125 disk.drbd2165.out‚0# disk.dm-0.out‚61217.86767578125 network.sent‚ 2.1516818e+07 network.recv‚ 8.266564e+06( disk.nvme0c0n1.out‚61217.86767578125" disk.nvme0n1.in‚44159.56640625& disk.nvme0n1.out‚61217.86767578125 disk.dm-1.out‚ 81.58203125) proc.memory.percent‚14.36917781829834 cpu‚27.243095873713447 disk.sda3.in‚40680.58203125 disk.sda.out‚ 81.58203125 disk.drbd2165.in‚0 disk.dm-0.in‚44159.56640625$ memory_percent‚68.60616207122803 proc.cpu.threads‚17) disk./.usagePercent‚32.71072036464976 disk.sda3.out‚ 81.58203125 disk.sda.in‚40680.58203125 disk.sda2.out‚0) proc.memory.availableMB‚ 5143.56640625# proc.memory.rssMB‚ 2354.24609375 ŒFÖ2æ&- _wandbruntime‚4396  _runtime‚4396;8&Ëç&Å _runtime‚4396.727679922 _step‚1361 ' lr-AdamW/pg1‚0.00011184254140625023 ' lr-AdamW/pg2‚0.00011184254140625023 trainer/global_step‚16549 % _timestamp‚1.7866630481975422e+09Ò²Ðè&Ê _step‚1361 ' lr-AdamW/pg1‚0.00011184254140625023 ' lr-AdamW/pg2‚0.00011184254140625023 trainer/global_step‚16549 % _timestamp‚1.7866630481975422e+09 _runtime‚4396.727679922Ñ Å Ëm2é&- _wandbruntime‚4396  _runtime‚4396I‘Å'ê&¿O YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚452995721076213 KDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚60 _step‚1362 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚397950500644352 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚ 116845667 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚41945088 JBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚665 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚61478912 IBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚82 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚209 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚322 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 4654602240 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 4002183743488 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚455675838431232 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚13888054 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚322 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚61289217 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 4628521206 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚41348020 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 5368709120 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚451673696632832 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 4552982528 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚55556452 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚13887974 F>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17104896 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚393718015787008 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 IBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚60 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 4232535186944 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 4552982528 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚451673713737728 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚322 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4289 JDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4876 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚52046735 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 845178880 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚393718036430848 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚262 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 4654602240 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚55560739 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 3993851502069 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚4287 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 4002141798400 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 4002183743488 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 122905600 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 125829120 F?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚60 UHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚70973440 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 _runtime‚4396.732901027 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚41945088 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚38158763 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 5368709120 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚429 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚455675897481216 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚451673713737728 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 @7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚5264 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚455675838431232 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚20643840 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚449001869574144 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚262 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚322 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 trainer/global_step‚16549 JDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17104896 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚393718015787008 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 IBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚60 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 4232535186944 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 4552982528 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚451673713737728 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚322 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4289 JDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4876 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚52046735 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 845178880 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚393718036430848 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚262 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 4654602240 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚55560739 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 3993851502069 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚4287 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 4002141798400 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 4002183743488 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 122905600 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 125829120 F?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚60 UHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚70973440 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 _runtime‚4396.732901027 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚41945088 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚38158763 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 5368709120 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚429 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚455675897481216 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚451673713737728 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 @7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚5264 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚455675838431232 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚20643840 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚449001869574144 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚262 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚322 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 trainer/global_step‚16549 JßœÏ&í&ÉM K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚52049753 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚13888865 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 5368709120 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 4654602240 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 122905600 I@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚4876 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚453022079054981 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 5494538240 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 _runtime‚4396.892105995 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚41948160 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚38160966 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚262 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚397973791802880 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 845178880 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚262 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚61292770 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚13888787 NADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚59053056 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 4552982528 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚449028038328320 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚41348068 PDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 _step‚1363 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚55564032 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚455702450890752 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚ 116852507 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 4232781066752 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 122306526 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚ 116852507 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 116856802 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚393741063159808 BDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚322 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚55559739 K>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59053056 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚41948160 E5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5264 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚61292768 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 122905600 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 4002414815232 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚4293 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654602240 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚451700036075520 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 831258624 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 3994082074729 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 4002372867072 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 5368709120 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 125829120 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚55559739 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 125829120 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚455702391837696 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚322 NADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚58387428 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚61292770 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚451700036075520 NGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚78 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 4232728643072 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚ 116856802 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 5368709120 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚61478912 D=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚60 EDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521206 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚322 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17104896 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 4002372867072 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17104896 G@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚60 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚429 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0·K·Ô&î&ÎM SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 122905600 I@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚4876 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚453022079054981 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 5494538240 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 _runtime‚4396.892105995 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚41948160 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚38160966 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚262 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚397973791802880 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 845178880 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚262 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚61292770 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚13888787 NADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚59053056 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 4552982528 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚449028038328320 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚41348068 PDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 _step‚1363 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚55564032 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚455702450890752 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚ 116852507 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 4232781066752 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 122306526 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚ 116852507 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 116856802 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚393741063159808 BDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚322 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚55559739 K>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59053056 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚41948160 E5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5264 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚61292768 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 122905600 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 4002414815232 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚4293 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654602240 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚451700036075520 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 831258624 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 3994082074729 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 4002372867072 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 5368709120 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 125829120 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚55559739 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 125829120 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚455702391837696 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚322 NADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚58387428 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚61292770 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚451700036075520 NGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚78 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 4232728643072 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚ 116856802 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 5368709120 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚61478912 D=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚60 EDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521206 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚322 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17104896 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 4002372867072 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17104896 G@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚60 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚429 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚52049753 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚13888865 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 5368709120 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 4654602240Ó -Îùø2ï&- _wandbruntime‚4396  _runtime‚4396üŸÎ,wð&ñ epoch‚41 trainer/global_step‚16549 _runtime‚4396.892796516 _step‚1364 1loss/FRt1/assignment_loss‚0.49409767470649546 1loss/FRt2/assignment_loss‚0.39127386344557574 0loss/FRt1/detection_loss‚0.45831160667651843 .loss/FRt2/detection_loss‚0.444603810748926 &loss/total_loss‚1.7882869555775156 % _timestamp‚1.7866630483628302e+09#ÊÙ–|ñ&ö % _timestamp‚1.7866630483628302e+09 epoch‚41 trainer/global_step‚16549 _runtime‚4396.892796516 _step‚1364 1loss/FRt1/assignment_loss‚0.49409767470649546 1loss/FRt2/assignment_loss‚0.39127386344557574 0loss/FRt1/detection_loss‚0.45831160667651843 .loss/FRt2/detection_loss‚0.444603810748926 &loss/total_loss‚1.7882869555775156Ô ’Vù2ò&- _wandbruntime‚4409  _runtime‚4409Þt†Éó&à trainer/global_step‚16599 % _timestamp‚1.7866630611702344e+09 _runtime‚4409.700255573 _step‚1365 & lr-AdamW/pg1‚0.0001087052540309932 & lr-AdamW/pg2‚0.0001087052540309932uÜÏ«Îô&È _runtime‚4409.700255573 _step‚1365 & lr-AdamW/pg1‚0.0001087052540309932 & lr-AdamW/pg2‚0.0001087052540309932 trainer/global_step‚16599 % _timestamp‚1.7866630611702344e+09Õ ýS<ã2õ&- _wandbruntime‚4409  _runtime‚4409ZróuÅ'ö&¿O PCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚58387380 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 4013728564224 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 122905600 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚52198096 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚450311159250944 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚452989813522432 KBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚4876 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 5368709120 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 @7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚5264 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 G>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4876 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4289 JDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚61466867 IBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚77 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚262 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 4654602240 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 4517085184 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 125829120 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17104896 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚13929083 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚399115211464704 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚322 _step‚1366 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 4654602240 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚452989813522432 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 4013770509312 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 KDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚60 C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚4289 PCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚59049984 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚457003542086656 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚394870405070848 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 5494538240 _runtime‚4409.704898435 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚452989830627328 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚38269013 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 845178880 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 M@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚59049984 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚4287 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚223 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 125829120 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚262 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚41945088 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 831258624 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 4005413312677 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚13929008 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚399115142588416 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚48232448 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 122306526 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚55721554 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 4013770509312 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚20643840 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 % _timestamp‚1.7866630611712067e+09 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 4013728564224 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚61466867 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚38269011 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚454316514176241 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 4005371964657 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 4552982528 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚41348020 F?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚60 trainer/global_step‚16599 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚457003542086656 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 JDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 % _timestamp‚1.7866630611712067e+09 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 4013728564224 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚61466867 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚38269011 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚454316514176241 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 4005371964657 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 4552982528 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚41348020 F?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚60 trainer/global_step‚16599 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚457003542086656 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 JDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4876 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4289 JDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚61466867 IBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚77 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚262 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 4654602240 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 4517085184 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 125829120 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17104896 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚13929083 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚399115211464704 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚322 _step‚1366Ö §p2ø&- _wandbruntime‚4409  _runtime‚4409¬*&ˆÏ&ù&ÉM TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 4517085184 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 5494538240 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚61470418 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚4293 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 845178880 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 4005602537269 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚457030154546176 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 122905600 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚209 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521206 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚55729134 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚ 117195259 NGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚62 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 122905600 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17104896 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚ 117195259 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚55724841 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚457030095493120 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 4245007181824 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚322 PDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 117199554 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚454342988929817 trainer/global_step‚16599 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 125829120 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚322 DDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59053056 H@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚665 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚61478912 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚453016152965120 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚453016152965120 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚38271216 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚322 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚450337345044480 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 5494538240 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚457030154546176 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚4293 G@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚60 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 _runtime‚4409.875258567 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚4295 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 4013959632896 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚13929913 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚399138503660544 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚429 SFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚64678912 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚394893431799808 D=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚60 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654602240 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚52201131 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚399138438981632 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚394893452443648 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚262 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚61470420 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚41348068 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 5494538240 IBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚60 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚55729134 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚55724841 G@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚64 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚38271218 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17104896 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 4014001581056 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 5368709120 NADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚59053056 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚61470418 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚52201067 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 125829120 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚262 _step‚1367 >5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5264 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 4552982528 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚322 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚223 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚61470420 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚450337328005120 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 4552982528 I@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚4876 NADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚58387428 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 125829120 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 4014001581056 EDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654602240 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚52201131 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚399138438981632 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚394893452443648 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚262 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚61470420 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚41348068 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 5494538240 IBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚60 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚55729134 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚55724841 G@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚64 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚38271218 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17104896 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 4014001581056 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 5368709120 NADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚59053056 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚61470418 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚52201067 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 125829120 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚262 _step‚1367 >5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5264 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 4552982528 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚322 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚223 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚61470420 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚450337328005120 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 4552982528 I@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚4876 NADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚58387428 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 125829120 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 4014001581056 EDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521206 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚55729134 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚ 117195259 NGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚62 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 122905600 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17104896 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚ 117195259 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚55724841 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚457030095493120 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 4245007181824 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚322 PDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 117199554 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚454342988929817 trainer/global_step‚16599 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 125829120 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚322 DDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59053056 H@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚665 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚61478912 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚453016152965120 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚453016152965120 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚38271216 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚322 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚450337345044480 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 5494538240 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚457030154546176 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚4293 G@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚60 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 _runtime‚4409.875258567 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚4295 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 4013959632896 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚13929913 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚399138503660544 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚429× Ðé×2û&- _wandbruntime‚4409  _runtime‚4409Ï~±tü&î 0loss/FRt1/assignment_loss‚0.5266575717503679 &loss/total_loss‚1.9330024935880143 trainer/global_step‚16599 $ _timestamp‚1.786663061345901e+09 _step‚1368 0loss/FRt2/assignment_loss‚0.4358196186896964 /loss/FRt1/detection_loss‚0.5167734707999546 /loss/FRt2/detection_loss‚0.4537518323479956 epoch‚41 _runtime‚4409.8757604053)¯·yý&ó trainer/global_step‚16599 $ _timestamp‚1.786663061345901e+09 _step‚1368 0loss/FRt2/assignment_loss‚0.4358196186896964 /loss/FRt1/detection_loss‚0.5167734707999546 /loss/FRt2/detection_loss‚0.4537518323479956 epoch‚41 _runtime‚4409.875760405 0loss/FRt1/assignment_loss‚0.5266575717503679 &loss/total_loss‚1.9330024935880143Ø c[ì þ&‚(:• •¡ùÓË”öà gpu.0.gpu‚36.0 gpu.0.memory‚5, gpu.0.memoryAllocated‚7.0313262939453125) gpu.0.memoryAllocatedBytes‚ 6039863296 gpu.0.temp‚35.0 gpu.0.powerWatts‚220.098' gpu.0.enforcedPowerLimitWatts‚500.0) gpu.0.powerPercent‚44.019600000000004 gpu.0.smClock‚1410 gpu.0.memoryClock‚1593$ gpu.0.correctedMemoryErrors‚1955# gpu.0.uncorrectedMemoryErrors‚0®_‘ÿ&‚(:† •¡ùÓÚ¢áâ disk.sda3.in‚40680.58203125 disk.sda2.in‚0 disk.drbd2165.in‚0 cpu‚27.242679070457363 proc.cpu.threads‚17) disk./.usagePercent‚32.71124159598947% disk./.usageGB‚487.42113876342773 disk.sda.in‚40680.58203125 disk.sda.out‚ 81.76171875" disk.nvme0n1.in‚44159.56640625! disk.nvme0n1.out‚ 61224.140625 disk.dm-1.out‚ 81.76171875 disk.drbd2165.out‚0$ disk.nvme0c0n1.in‚44159.56640625# disk.nvme0c0n1.out‚ 61224.140625 disk.sda1.in‚0 disk.sda1.out‚0 disk.dm-0.out‚ 61224.140625$ memory_percent‚68.60485076904297) proc.memory.percent‚14.36917781829834 network.sent‚ 2.1584033e+07 network.recv‚ 8.290705e+06 disk.sda3.out‚ 81.76171875 disk.sda2.out‚0 disk.dm-1.in‚40680.58203125 disk.dm-0.in‚44159.56640625& proc.memory.availableMB‚ 5143.78125# proc.memory.rssMB‚ 2354.24609375C©2€'- _wandbruntime‚4422  _runtime‚442223aË'Å _runtime‚4422.348927032 _step‚1369 ' lr-AdamW/pg1‚0.00010560915581891297 ' lr-AdamW/pg2‚0.00010560915581891297 trainer/global_step‚16649 % _timestamp‚1.7866630738163195e+09™uN¸Ð‚'Ê ' lr-AdamW/pg1‚0.00010560915581891297 ' lr-AdamW/pg2‚0.00010560915581891297 trainer/global_step‚16649 % _timestamp‚1.7866630738163195e+09 _runtime‚4422.348927032 _step‚1369Ù 7i0E2ƒ'- _wandbruntime‚4422  _runtime‚4422x£òð‰„'¾O VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 4517085184 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚55886656 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚4287 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚41945088 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚400279858654720 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 4016933775265 IBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚60 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 4654602240 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚209 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚322 N@DeviceStatsMonitor.on_train_¨¨;#batch_start/allocation.all.allocated‚ 117535460 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 _step‚1370 G>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4876 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚20643840 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚455637365663649 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚458331245742080 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚454305947516928 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚55886656 UHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚64681984 PCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚58387380 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚38379263 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 122306526 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 5494538240 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚61644515 M@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚59049984 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 831258624 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚458331304792064 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚55890943 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 4654602240 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 4552982528 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 125829120 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 4025315330048 D;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚5264 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚400279793972736 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 125829120 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 125829120 F>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚41945088 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4289 JDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚262Ì¡_É'…'ÃO C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚4289 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚41348020 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚262 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚223 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 5368709120 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 122905600 @7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚5264 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 4025315330048 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 845178880 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 4552982528 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 122905600 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 JBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚665 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚262 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 4517085184 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚55886656 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚4287 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚41945088 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚400279858654720 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 4016933775265 IBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚60 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 4654602240 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚209 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚322 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚ 117535460 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 _step‚1370 G>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4876 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚20643840 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚455637365663649 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚458331245742080 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚454305947516928 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚55886656 UHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚64681984 PCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚58387380 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚38379263 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 122306526 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 5494538240 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚61644515 M@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚59049984 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 831258624 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚458331304792064 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚55890943 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 4654602240 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 4552982528 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 125829120 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 4025315330048 D;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚5264 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚400279793972736 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 125829120 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 125829120 F>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚41945088 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4289 JDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚52352301 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 5494538240 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚55894236 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚39840768 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521206 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 4552982528 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚322 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 4025588346880 DDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 4517085184 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚454332269854720 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 >5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5264 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚ 117538011 H@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚665 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 122306526 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚322 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 B9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚5264 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚429 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚396045800439808 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 831258624 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚41348068 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚38381466 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 4025546398720 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚61648070 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚458357858201600 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚396045821083648 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚451646617681920 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 5368709120 _step‚1371 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚458357858201600 NADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚59053056 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 K>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59053056 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 4654602240 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17104896 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 5494538240 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 4025588346880 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 122905600 EDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654602240 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 4552982528 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 trainer/global_step‚16649 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚4293 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚400303149810176 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚ 117542306 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚61648068 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚55889943 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 4025546398720 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 125829120 G@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚60 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚55894236 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚4293 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚454332252749824 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 125829120 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚455663782029797 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 _runtime‚4422.507890494 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚41948160 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 117542306 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚52352369 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚61478912 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚455663840417225 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 5494538240 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚ 117538011 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚454332269854720 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚13970835 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 845178880 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 4257288885760 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚322 SFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚60484608 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 4017205695945 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 5368709120 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚262 EDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654602240 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 4552982528 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 trainer/global_step‚16649 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚4293 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚400303149810176 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚ 117542306 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚61648068 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚55889943 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 4025546398720 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 125829120 G@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚60 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚55894236 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚4293 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚454332252749824 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 125829120 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚455663782029797 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 _runtime‚4422.507890494 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚41948160 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 117542306 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚52352369 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚61478912 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚455663840417225 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 5494538240 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚ 117538011 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚454332269854720 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚13970835 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 845178880 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 4257288885760 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚322 SFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚60484608 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 4017205695945 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 5368709120 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚262 EDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚52352301 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 5494538240 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚55894236 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚39840768 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521206 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 4552982528 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚322 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 4025588346880 DDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 4517085184 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚454332269854720 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 >5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5264 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚ 117538011 H@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚665 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 122306526 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚322 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 B9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚5264 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚429 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚396045800439808 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 831258624 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚41348068 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚38381466 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 4025546398720 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚61648070 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚458357858201600 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚396045821083648 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚451646617681920 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 5368709120 _step‚1371 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚458357858201600 NADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚59053056 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 K>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59053056 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 4654602240 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17104896 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 5494538240 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 4025588346880 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 122905600 EÙ¡'‚(:– ¤¡ùÓß­àâ network.sent‚ 2.1676403e+07 network.recv‚ 8.319713e+06 disk.sda1.in‚0" disk.nvme0n1.in‚44167.82421875 disk.dm-1.out‚ 81.95703125' proc.memory.availableMB‚ 5143.453125 proc.cpu.threads‚17) disk./.usagePercent‚32.71124518009495 disk.drbd2165.in‚0 disk.dm-0.in‚44167.82421875# disk.dm-0.out‚61268.25634765625$ disk.nvme0c0n1.in‚44167.82421875 disk.sda2.out‚0# proc.memory.rssMB‚ 2354.24609375) proc.memory.percent‚14.36917781829834 disk.sda3.out‚ 81.95703125( disk.nvme0c0n1.out‚61268.25634765625 disk.sda.in‚40724.30859375 disk.sda1.out‚0 disk.sda2.in‚0& disk.nvme0n1.out‚61268.25634765625$ memory_percent‚68.60685348510742 cpu‚27.242367578472305 disk.sda.out‚ 81.95703125% disk./.usageGB‚487.42119216918945 disk.drbd2165.out‚0 disk.sda3.in‚40724.30859375 disk.dm-1.in‚40724.30859375ŠýÐ2Ž'- _wandbruntime‚4434  _runtime‚4434:á°°Ë'Å ' lr-AdamW/pg1‚0.00010255444562661203 ' lr-AdamW/pg2‚0.00010255444562661203 trainer/global_step‚16699 % _timestamp‚1.7866630863262777e+09 _runtime‚4434.856259584 _step‚1373Í‚°¿Ð'Ê ' lr-AdamW/pg2‚0.00010255444562661203 trainer/global_step‚16699 % _timestamp‚1.7866630863262777e+09 _runtime‚4434.856259584 _step‚1373 ' lr-AdamW/pg1‚0.00010255444562661203Ý gÏó2‘'- _wandbruntime‚4434  _runtime‚4434±v¾SÄ'’'¾O C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚4289 BDeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4289 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚322 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 4036944040960 JBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚665 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 5368709120 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 125829120 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 4036902095872 PIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚71 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚456958275538437 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚322 _runtime‚4434.857426831 F>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚4287 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚38489511 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚223 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚397175142350848 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 PCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚58387380 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚452929738604544 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚56051758 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚455622047301632 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚14011028 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 845178880 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 5494538240 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚322 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚41945088 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚209 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚401444532527104 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 4269344040960 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 125829120 KDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚60 F?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚60 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚56056045 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚459658949397504 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚ 117878212 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚455622047301632 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚46135296 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 5494538240 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 4269390176256 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 IBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚60 trainer/global_step‚16699 @7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚5264 G>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4876 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚61478912 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 4552982528 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚56056045 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚455622064406528 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚41945088 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚52500539 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚429 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚14011099 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚452929721565184 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚262 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚ 117873923 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚61822167 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 4028536933893 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 4517085184 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 122306526 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 5368709120 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚459659008447488 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17104896 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚52500612 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚20643840 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 4654602240 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 4552982528 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 122905600 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚397175121707008 $ _timestamp‚1.786663086327181e+09 JDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚459658949397504 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 4036944040960 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚61822167 UHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚66779136 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 4028495585873 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚455622064406528 _step‚1374 M@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚59049984 D;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚5264 KBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚4876 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 5494538240 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 5368709120 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚262 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚4287 JDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4876 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚61478912 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 4552982528 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚56056045 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚455622064406528 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚41945088 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚52500539 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚429 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚14011099 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚452929721565184 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚262 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚ 117873923 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚61822167 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 4028536933893 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 4517085184 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 122306526 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 5368709120 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚459659008447488 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17104896 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚52500612 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚20643840 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 4654602240 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 4552982528 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 122905600 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚397175121707008 $ _timestamp‚1.786663086327181e+09 JDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚459658949397504 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 4036944040960 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚61822167 UHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚66779136 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 4028495585873 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚455622064406528 _step‚1374 M@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚59049984 D;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚5264 KBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚4876 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 5494538240 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 5368709120 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚262 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚4287 JDeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4289 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚322 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 4036944040960 JBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚665 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 5368709120 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 125829120 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 4036902095872 PIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚71 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚456958275538437 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚322 _runtime‚4434.857426831 F>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚4287 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚38489511 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚223 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚397175142350848 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 PCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚58387380 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚452929738604544 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚56051758 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚455622047301632 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚14011028 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 845178880 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 5494538240 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚322 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚41945088 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚209 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚401444532527104 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 4269344040960 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 125829120 KDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚60 F?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚60 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚56056045 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚459658949397504 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚ 117878212 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚455622047301632 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚46135296 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 5494538240Þ ^õ>2”'- _wandbruntime‚4435  _runtime‚4435‡êG· •'ÉM J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 4654602240 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17104896 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚223 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚20643840 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚61825718 NADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚59053056 G@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚68 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚61478912 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 4552982528 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚397198169079808 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚61825720 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 125829120 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚ 117885058 _runtime‚4435.018826642 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚56059338 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 4269589923840 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 5494538240 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 5368709120 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 PDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚459685502803968 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 4037133164544 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 117885058 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚38491718 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 EDeviceStatsMonito.¨”Hr.on_train_batch_end/active_bytes.all.current‚59053056 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚455648386744320 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚455648369639424 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚41948160 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚39840768 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 B6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚322 G@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚60 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚61825720 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚56055045 SFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚60484608 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚401467759003648 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚397198189723648 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 4517085184 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚456984691904633 EDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚52503586 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚322 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 4028726158485 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚4295 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚14011936 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 125829120 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚429 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚14011870 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 D8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚456984633517205 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 4028767506553 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 4037133164544 B9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚5264 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚262 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚459685561857024 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚41948160 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 125829120 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚ 117880763 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚401467819488256 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521206 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚209 D=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚60 >5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5264 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚56059338 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654602240 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17104896 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚52503654 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 831258624 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 4269629764608 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚455648386744320 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 5494538240 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚322 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 % _timestamp‚1.7866630864884615e+09è[ˆÔ&–'ÎM H@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚665 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚52503586 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚322 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 4028726158485 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚4295 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚14011936 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 125829120 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚429 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚14011870 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 D8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚456984633517205 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 4028767506553 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 4037133164544 B9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚5264 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚262 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚459685561857024 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚41948160 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 125829120 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚ 117880763 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚401467819488256 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521206 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚209 D=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚60 >5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5264 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚56059338 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654602240 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17104896 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚52503654 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 831258624 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 4269629764608 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚455648386744320 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 5494538240 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚322 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 % _timestamp‚1.7866630864884615e+09 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 4654602240 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17104896 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚223 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚20643840 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚61825718 NADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚59053056 G@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚68 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚61478912 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 4552982528 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚397198169079808 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚61825720 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 125829120 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚ 117885058 _runtime‚4435.018826642 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚56059338 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 4269589923840 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 5494538240 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 5368709120 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 PDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚459685502803968 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 4037133164544 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 117885058 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚38491718 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 EDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59053056 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚455648386744320 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚455648369639424 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚41948160 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚39840768 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 B6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚322 G@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚60 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚61825720 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚56055045 SFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚60484608 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚401467759003648 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚397198189723648 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 4517085184 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚456984691904633 EDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4876 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 4048530806784 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚460986653052928 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚456938181296128 PIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚82 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 4281677541376 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 PCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚59049984 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚322 _runtime‚ 4447.6658676 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17104896 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚458279068638465 JBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚665 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚41348020 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 5368709120 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚4289 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚456938164191232 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚262 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17104896 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 5494538240 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 125829120 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 4654602240 KBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚4876 UHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚68876288 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 4552982528 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 4048530806784 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚429 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 4040098744501 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 5368709120 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚456938181296128 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚322 IBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚84 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 845178880 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚454239011241984 @7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚5264 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 KDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚60 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚262 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚262 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚38599761 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚223 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚61999815 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚56221147 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 122905600 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4289 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚14051863 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚460986712102912 D;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚5264 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚52651542 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚402609119655936 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 4628521206 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚ 118216675 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚56216860 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 4048488861696 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 4654602240 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚20643840 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 831258624 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚61478912 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 4040057396481 F>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚56221147 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚48232448 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚322 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 F?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚60 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚4287 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 5494538240 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚454239028281344 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 5368709120 IBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚60 JDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚460986653052928 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚460986712102912 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚402609188532224 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 % _timestamp‚1.7866630991326785e+09 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚456938164191232 JDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4876 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 4048530806784 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚460986653052928 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚456938181296128 PIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚82 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 4281677541376 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 PCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚59049984 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚322 _runtime‚ 4447.6658676 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17104896 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚458279068638465 JBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚665 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚41348020 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 5368709120 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚4289 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚456938164191232 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚262 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17104896 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 5494538240 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 125829120 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 4654602240 KBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚4876 UHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚68876288 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 4552982528 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 4048530806784 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚429 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 4040098744501 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 5368709120 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚456938181296128 CÖCìÂ^;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚322 IBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚84 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 845178880 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚454239011241984 @7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚5264 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 KDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚60 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚262 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚262 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚38599761 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚223 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚61999815 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚56221147 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 122905600 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4289 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚14051863 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚460986712102912 D;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚5264 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚52651542 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚402609119655936 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 4628521206 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚ 118216675 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚56216860 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 4048488861696 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 4654602240 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚20643840 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 831258624 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚61478912 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 4040057396481 F>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚56221147 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚48232448 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚322 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 F?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚60 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚4287 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 5494538240 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚454239028281344 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 5368709120 IBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚60 JDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚460986653052928 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚460986712102912 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚402609188532224 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 % _timestamp‚1.7866630991326785e+09 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚456938164191232 JDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59053056 EDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚52654576 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚14052685 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 4048761878528 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚209 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 _runtime‚4447.829788582 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 G@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚77 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚458305543392041 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚56224440 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚454265214074880 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚ 118227810 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚4293 IBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚60 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654602240 EDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521206 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 5494538240 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 125829120 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚262 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚41948160 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 122905600 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚456964486529024 I@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚4876 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 4040329317161 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 122306526 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 4517085184 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚322 NADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚59053056 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚461013206459392 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚398350558363648 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚398350537719808 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 4654602240 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚461013265512448 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚48229376 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 4281875191808 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚322 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚429 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 5368709120 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚456964503633920 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚41948160 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 118227810 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚ 118223515 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚20643840 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 4552982528 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 125829120 D=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚60 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚461013265512448 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 4552982528 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚62003370 NGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚75 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 831258624 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 122905600 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚62003368 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 5494538240 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚262 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚62003368 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚56220147 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚14052610 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 5368709120 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 125829120 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚322 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚322 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 845178880 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚454265197035520 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚456964503633920 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚402632481784832 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 >5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5264 H@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚665 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚223 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚61478912 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 B6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚322 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 845178880 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚454265197035520 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚456964503633920 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚402632481784832 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 >5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5264 H@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚665 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚223 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚61478912 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 BDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59053056 EDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚52654576 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚14052685 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 4048761878528 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚209 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 _runtime‚4447.829788582 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 G@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚77 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚458305543392041 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚56224440 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚454265214074880 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚ 118227810 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚4293 IBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚60 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654602240 EDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521206 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 5494538240 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 125829120 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚262 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚41948160 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 122905600 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚456964486529024 I@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚4876 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 4040329317161 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 122306526 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 4517085184 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚322 NADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚59053056 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚461013206459392 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚398350558363648 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚398350537719808 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 4654602240 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚461013265512448 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚48229376 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 4281875191808 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚322 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚429 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 5368709120 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚456964503633920 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚41948160 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 118227810 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚ 118223515 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚20643840 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 4552982528 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 125829120 D=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚60 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚461013265512448 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 4552982528 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚62003370 NGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚75 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 831258624 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 122905600 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚62003368 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0ã ´¿‹Á2¥'- _wandbruntime‚4447  _runtime‚44473säv¦'ð /loss/FRt1/detection_loss‚0.5131284145551024 %loss/total_loss‚1.849575329116478 $ _timestamp‚1.786663099300363e+09 _runtime‚4447.830198006 1loss/FRt1/assignment_loss‚0.48003349109887433 0loss/FRt2/detection_loss‚0.47668308913051427 epoch‚41 trainer/global_step‚16749 _step‚1380 1loss/FRt2/assignment_loss‚0.37973033433198716‹Á{§'õ 1loss/FRt1/assignment_loss‚0.48003349109887433 0loss/FRt2/detection_loss‚0.47668308913051427 epoch‚41 trainer/global_step‚16749 _step‚1380 1loss/FRt2/assignment_loss‚0.37973033433198716 /loss/FRt1/detection_loss‚0.5131284145551024 %loss/total_loss‚1.849575329116478 $ _timestamp‚1.786663099300363e+09 _runtime‚4447.830198006ä Ê1–¨'‚(:‹ ¡ùÓߘá gpu.0.gpu‚62.0 gpu.0.memory‚11, gpu.0.memoryAllocated‚7.0313262939453125) gpu.0.memoryAllocatedBytes‚ 6039863296 gpu.0.temp‚37.0 gpu.0.powerWatts‚207.162' gpu.0.enforcedPowerLimitWatts‚500.0 gpu.0.powerPercent‚41.4324 gpu.0.smClock‚1410 gpu.0.memoryClock‚1593$ gpu.0.correctedMemoryErrors‚1955# gpu.0.uncorrectedMemoryErrors‚0$e·àœ©'‚(:‘ ¡ùÓ ËÃã network.recv‚ 8.372041e+06 disk.dm-1.in‚ 41893.640625 disk.sda.in‚ 41893.640625 disk.sda2.out‚0" disk.nvme0n1.in‚44180.16015625& disk.nvme0n1.out‚61319.66259765625# disk.dm-0.out‚61319.66259765625 disk.sda3.in‚ 41893.640625 disk.sda3.out‚ 82.83203125# memory_percent‚68.6058759689331# proc.memory.rssMB‚ 2354.24609375 cpu‚27.241574022671916 network.sent‚ 2.1814827e+07) disk./.usagePercent‚32.72802749799454% disk./.usageGB‚487.67126083374023 disk.drbd2165.in‚0 disk.dm-0.in‚44180.16015625( disk.nvme0c0n1.out‚61319.66259765625 disk.sda.out‚ 82.83203125 proc.cpu.threads‚17 disk.dm-1.out‚ 82.83203125 disk.drbd2165.out‚0$ disk.nvme0c0n1.in‚44180.16015625 disk.sda1.in‚0 disk.sda1.out‚0 disk.sda2.in‚0) proc.memory.availableMB‚ 5143.61328125) proc.memory.percent‚14.36917781829834žCÔÓ2ª'- _wandbruntime‚4460  _runtime‚4460ÙÓ^¿Ç«'Á & lr-AdamW/pg1‚9.656997142378086e-05 & lr-AdamW/pg2‚9.656997142378086e-05 trainer/global_step‚16799 $ _timestamp‚1.786663111734535e+09 _runtime‚ 4460.26459802 _step‚1381ÔÁö¤Ì¬'Æ & lr-AdamW/pg2‚9.656997142378086e-05 trainer/global_step‚16799 $ _timestamp‚1.786663111734535e+09 _runtime‚ 4460.26459802 _step‚1381 & lr-AdamW/pg1‚9.656997142378086e-05å e‚¾É2­'- _wandbruntime‚4460  _runtime‚44600x±é ®'¿O WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 @7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚5264 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚41945088 IBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚68 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚399479879630848 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 122306526 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 5368709120 trainer/global_step‚16799 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚462314415758336 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚52802474 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚399479858987008 IBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚60 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚4287 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚56381962 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚20643840 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 JDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4289 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚56381962 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 5494538240 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚322 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 4552982528 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚462314415758336 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚322 % _timestamp‚1.7866631117356577e+09 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚462314356708352 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚62177465 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚56386249 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 125829120 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚62177467 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚38710013 UHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚70973440 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚403773771414528 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 122905600 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚62177467 KBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚4876 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚209 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚61478912 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚455548317958144 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 4051660555109 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 5368709120 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚462314356708352 PIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚66 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 831258624 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 5494538240 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚459599920125873 G>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4876 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 845178880 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 KDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚60 _step‚1382 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 4060075627520 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚41945088 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚41348020 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚322 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚458254281080832 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 122905600 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚50329600 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚458254298185728 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 4293962757120 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 5368709120 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚62177465 M@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚59049984 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚458254281080832 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17104896 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚223 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚455548300918784 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚262 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚4287 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0„N¨}Ê'¯'ÄO JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚4287 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 @7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚5264 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚41945088 IBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚68 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚399479879630848 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 122306526 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 5368709120 trainer/global_step‚16799 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚462314415758336 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚52802474 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚399479858987008 IBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚60 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚4287 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚56381962 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚20643840 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 JDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4289 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚56381962 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 5494538240 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚322 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 4552982528 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚462314415758336 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚322 % _timestamp‚1.7866631117356577e+09 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚462314356708352 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚62177465 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚56386249 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 125829120 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚62177467 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚38710013 UHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚70973440 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚403773771414528 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 122905600 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚62177467 KBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚4876 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚209 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚61478912 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚455548317958144 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 4051660555109 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 5368709120 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚462314356708352 PIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚66 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 831258624 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 5494538240 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚459599920125873 G>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4876 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 845178880 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 KDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚60 _step‚1382 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 4060075627520 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚41945088 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚41348020 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚322 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚458254281080832 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 122905600 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚50329600 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚458254298185728 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 4293962757120 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 5368709120 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚62177465 M@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚59049984 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚458254281080832 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17104896 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚223 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚455548300918784 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚262æ šªM/2°'- _wandbruntime‚4460  _runtime‚44607”¹nÏ&±'ÉM UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚41348068 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 4051849779701 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 4294155033600 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 5368709120 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 5494538240 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚262 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚4293 SFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚68873216 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 831258624 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 4517085184 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 5368709120 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 118570562 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚459626336492021 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚62181020 H@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚665 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚38712218 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 5494538240 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 5368709120 D=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚60 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚458280603418624 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚56389542 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚429 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚56385249 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚462340969167872 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 125829120 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚62181018 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚458280620523520 G@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚64 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚399502927003648 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚62181020 DDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654602240 K>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59053056 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚458280620523520 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 4552982528 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚459626394879449 E8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 NADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚58387428 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚56389542 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 4060348644352 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚458280603418624 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 G@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚60 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚52805435 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚38712216 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚262 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚262 trainer/global_step‚16799 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17104896 B9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚5264 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚56385249 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚403797130266624 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 845178880 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 125829120 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚322 IBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚60 _step‚1383 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚4295 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚14093281 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚20643840 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 B5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5264 EDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521206 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚403797061393408 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 4654602240 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 4060306696192 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 5494538240 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚322 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚399502906359808 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚455574503751680 P6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚322 NADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚59053056 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚209 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 % _timestamp‚1.7866631119095824e+09 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚ 118570562 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 4060306696192 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚48229376 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚462340910114816 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 4060348644352 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚ 118566267 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 125829120 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚223ÞíÝ7l²'ÎM A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚4295 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚14093281 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚20643840 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 B5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5264 EDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521206 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚403797061393408 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 4654602240 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 4060306696192 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 5494538240 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚322 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚399502906359808 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚455574503751680 P6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚322 NADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚59053056 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚209 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 % _timestamp‚1.7866631119095824e+09 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚ 118570562 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 4060306696192 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚48229376 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚462340910114816 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 4060348644352 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚ 118566267 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 125829120 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚223 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚41348068 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 4051849779701 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 4294155033600 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 5368709120 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 5494538240 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚262 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚4293 SFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚68873216 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 831258624 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 4517085184 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 5368709120 L>DeviceStatsMonitor.on_train_batch_en0Ýb™hd/allocation.all.allocated‚ 118570562 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚459626336492021 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚62181020 H@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚665 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚38712218 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 5494538240 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 5368709120 D=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚60 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚458280603418624 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚56389542 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚429 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚56385249 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚462340969167872 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 125829120 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚62181018 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚458280620523520 G@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚64 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚399502927003648 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚62181020 DDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654602240 K>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59053056 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚458280620523520 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 4552982528 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚459626394879449 E8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 NADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚58387428 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚56389542 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 4060348644352 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚458280603418624 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 G@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚60 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚52805435 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚38712216 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚262 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚262 trainer/global_step‚16799 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17104896 B9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚5264 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚56385249 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚403797130266624 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 845178880 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 125829120 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚322 IBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚60 _step‚1383 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1ç îÝÔe2³'- _wandbruntime‚4460  _runtime‚4460 +lw´'ñ /loss/FRt2/detection_loss‚0.4858349525912507 &loss/total_loss‚1.8393444453058274 epoch‚41 % _timestamp‚1.7866631119108698e+09 _runtime‚4460.440806787 _step‚1384 1loss/FRt2/assignment_loss‚0.37606212994105404 /loss/FRt1/detection_loss‚0.4829624797162368 trainer/global_step‚16799 1loss/FRt1/assignment_loss‚0.49448488305728594#ÔÒÿ|µ'ö /loss/FRt2/detection_loss‚0.4858349525912507 &loss/total_loss‚1.8393444453058274 epoch‚41 % _timestamp‚1.7866631119108698e+09 _runtime‚4460.440806787 _step‚1384 1loss/FRt2/assignment_loss‚0.37606212994105404 /loss/FRt1/detection_loss‚0.4829624797162368 trainer/global_step‚16799 1loss/FRt1/assignment_loss‚0.49448488305728594è â`Ÿá2¶'- _wandbruntime‚4465  _runtime‚4465ní ×ß·'Ù 3Purity/1FRt/event_proportion‚0.6297432780265808 *jet/accuracy_1_of_2‚0.5774189829826355 +particle/sensitivity‚0.6296482086181641 /Purity/1FRt/event_purity‚0.4673014283180237 -Purity/2FRt/FRt_purity‚0.5654901266098022 +validation_accuracy‚0.21208564937114716 *loss/val_total_loss‚2.5153611617816005 particle/accuracy_1_of_1‚1 /particle/accuracy_1_of_2‚0.9218981862068176 +particle/specificity‚0.6472073793411255 6validation_average_jet_accuracy‚0.3154866099357605 % _timestamp‚1.7866631172692156e+09 _runtime‚4465.799308962 /particle/accuracy_2_of_2‚0.6781596541404724 trainer/global_step‚16799 /Purity/*FRt/event_purity‚0.4651513397693634 .Purity/*FRt/FRt_purity‚0.47855594754219055 3Purity/0FRt/event_proportion‚0.3301943838596344 2Purity/2FRt/event_proportion‚0.040062315762043 'particle/f_score‚0.5545425415039062 _step‚1385 -Purity/1FRt/FRt_purity‚0.4673014283180237 *jet/accuracy_1_of_1‚0.3103941082954407 'particle/accuracy‚0.640985369682312 Purity/0FRt/event_purity‚1 0Purity/2FRt/event_purity‚0.42918509244918823 +jet/accuracy_2_of_2‚0.21208564937114716 epoch‚41 2Purity/*FRt/event_proportion‚0.669805645942688‚þ?ä¸'Þ 2Purity/*FRt/event_proportion‚0.669805645942688 3Purity/1FRt/event_proportion‚0.6297432780265808 *jet/accuracy_1_of_2‚0.5774189829826355 +particle/sensitivity‚0.6296482086181641 /Purity/1FRt/event_purity‚0.4673014283180237 -Purity/2FRt/FRt_purity‚0.5654901266098022 +validation_accuracy‚0.21208564937114716 *loss/val_total_loss‚2.5153611617816005 particle/accuracy_1_of_1‚1 /particle/accuracy_1_of_2‚0.9218981862068176 +particle/specificity‚0.6472073793411255 6validation_average_jet_accuracy‚0.3154866099357605 % _timestamp‚1.7866631172692156e+09 _runtime‚4465.799308962 /particle/accuracy_2_of_2‚0.6781596541404724 trainer/global_step‚16799 /Purity/*FRt/event_purity‚0.4651513397693634 .Purity/*FRt/FRt_purity‚0.47855594754219055 3Purity/0FRt/event_proportion‚0.3301943838596344 2Purity/2FRt/event_proportion‚0.040062315762043 'particle/f_score‚0.5545425415039062 _step‚1385 -Purity/1FRt/FRt_purity‚0.4673014283180237 *jet/accuracy_1_of_1‚0.3103941082954407 'particle/accuracy‚0.640985369682312 Purity/0FRt/event_purity‚1 0Purity/2FRt/event_purity‚0.42918509244918823 +jet/accuracy_2_of_2‚0.21208564937114716 epoch‚41é ˜ ³ ¹'‚(:• Ñ¡ùÓßÝâ gpu.0.gpu‚51.0 gpu.0.memory‚9, gpu.0.memoryAllocated‚7.0313262939453125) gpu.0.memoryAllocatedBytes‚ 6039863296 gpu.0.temp‚37.0 gpu.0.powerWatts‚187.538' gpu.0.enforcedPowerLimitWatts‚500.0) gpu.0.powerPercent‚37.507600000000004 gpu.0.smClock‚1410 gpu.0.memoryClock‚1593$ gpu.0.correctedMemoryErrors‚1955# gpu.0.uncorrectedMemoryErrors‚0 Bû›º'‚(: Ñ¡ùÓƒœ‹æ disk.sda3.out‚ 82.99609375( disk.nvme0c0n1.out‚61368.66552734375 disk.sda.out‚ 82.99609375 disk.sda1.out‚0 disk.sda2.out‚0& disk.nvme0n1.out‚61368.66552734375$ memory_percent‚68.60480308532715 disk.drbd2165.out‚0" disk.nvme0n1.in‚44182.12109375# proc.memory.rssMB‚ 2354.24609375) proc.memory.percent‚14.36917781829834 network.sent‚ 2.1886193e+07) disk./.usagePercent‚32.72803364217536 disk.drbd2165.in‚0 disk.sda3.in‚ 41893.640625 disk.sda2.in‚0( proc.memory.availableMB‚ 5143.7890625 cpu‚27.247393973836424 proc.cpu.threads‚17 network.recv‚ 8.400922e+06$ disk./.usageGB‚487.6713523864746 disk.dm-1.in‚ 41893.640625 disk.dm-1.out‚ 82.99609375# disk.dm-0.out‚61368.66552734375$ disk.nvme0c0n1.in‚44182.12109375 disk.sda.in‚ 41893.640625 disk.sda1.in‚0 disk.dm-0.in‚44182.12109375¯\òÑ2»'- _wandbruntime‚4478  _runtime‚4478ã!ɼ'à % _timestamp‚1.7866631299124992e+09 _runtime‚4478.442504243 _step‚1386 & lr-AdamW/pg1‚9.364059178484608e-05 & lr-AdamW/pg2‚9.364059178484608e-05 trainer/global_step‚16849¦S$žÎ½'È _step‚1386 & lr-AdamW/pg1‚9.364059178484608e-05 & lr-AdamW/pg2‚9.364059178484608e-05 trainer/global_step‚16849 % _timestamp‚1.7866631299124992e+09 _runtime‚4478.442504243ê 3ÅY2¾'- _wandbruntime‚4478  _runtime‚4478ü±±¯Å'¿'¿O UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚464037065302528 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚ 118989261 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚62415721 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚56569251 PIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚80 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 PCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚59049984 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚56573538 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 5494538240 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 trainer/global_step‚16849 M@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚59049984 KBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚4876 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚405241853671424 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚61478912 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚461305737077222 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 5494538240 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚459961352290304 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚53002829 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 4067246989798 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚322 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚322 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚46135296 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 4310288957440 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 5494538240 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 125829120 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚459961352290304 JBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚665 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 4067288337818 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 BDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚464037065302528 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚41945088 D;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚5264 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚459961335185408 JDeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4289 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚14141952 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚14141872 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 5368709120 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚322 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚62415721 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚464037124352512 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚405241786892288 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 831258624 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚56573538 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚4287 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚461305795464602 PCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚58387380 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 122306526 JDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4876 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚20643840 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 F?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚60 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 4628521206 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 125829120 C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚4289 F>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 4075730117120 IBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚82 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚459961335185408 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 4075730117120 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚429 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚457238507126784 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 125829120 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚400931518578688 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 4310335092736 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚322 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚ 118984972 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2ÒfB„Ê'À'ÄO @7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚5264 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚464037065302528 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚41945088 D;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚5264 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚459961335185408 JDeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4289 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚14141952 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚14141872 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 5368709120 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚322 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚62415721 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚464037124352512 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚405241786892288 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 831258624 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚56573538 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚4287 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚461305795464602 PCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚58387380 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 122306526 JDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4876 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚20643840 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 F?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚60 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 4628521206 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 125829120 C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚4289 F>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 4075730117120 IBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚82 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚459961335185408 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 4075730117120 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚429 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚457238507126784 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 125829120 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚400931518578688 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 4310335092736 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚322 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚ 118984972 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚464037065302528 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚ 118989261 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚62415721 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚56569251 PIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚80 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 PCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚59049984 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚56573538 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 5494538240 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 trainer/global_step‚16849 M@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚59049984 KBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚4876 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚405241853671424 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚61478912 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚461305737077222 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 5494538240 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚459961352290304 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚53002829 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 4067246989798 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚322 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚322 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚46135296 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 4310288957440 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 5494538240 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 125829120 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚459961352290304 JBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚665 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 4067288337818 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 BDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 4310577138688 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 125829120 EDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 $ _timestamp‚1.786663130080812e+09 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 118996107 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚38863082 D=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚60 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚62419276 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚459987674628096 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 4067518910478 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚464063677762048 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17104896 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚322 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚461332211830798 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚322 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚56576831 K>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59053056 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚405265143090176 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚457264675880960 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚ 118991812 C=DeviceStatsMonitor.on_nšd>¾train_batch_end/oversize_segments.freed‚0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 5368709120 IBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚60 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚62419274 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚41948160 H@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚665 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 4067477562410 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚322 G@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚60 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚464063677762048 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 5494538240 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 4552982528 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚429 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚14142756 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 trainer/global_step‚16849 D5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5264 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 4654602240 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 4075961185792 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚56572538 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚14142696 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 125829120 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚62419274 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚53005838 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚ 118996107 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚459987674628096 B9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚5264 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚56576831 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚38863080 G@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚62 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 122306526 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 _step‚1388 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚41948160 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚4293 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚61478912 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521206 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚322 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚4295 NADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚59053056 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 4075961185792 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 845178880 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚400954565951488 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 5494538240 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 5368709120 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚53005776 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚459987657523200 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 _runtime‚4478.611199629 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚223 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚459987657523200 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚457264692920320 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 4517085184 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚262 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚4293 EDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654602240 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚20643840 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 125829120 I@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚4876 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 122905600 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚62419276 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 4310539395072 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 4076003133952Ôaû=Ó&Ã'ÍM LDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚223 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚459987657523200 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚457264692920320 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 4517085184 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚262 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚4293 EDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654602240 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚20643840 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 125829120 I@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚4876 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 122905600 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚62419276 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 4310539395072 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 4076003133952 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17104896 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚209 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚41348068 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 4310577138688 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 125829120 EDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 $ _timestamp‚1.786663130080812e+09 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 118996107 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚38863082 D=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚60 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚62419276 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚459987674628096 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 4067518910478 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚464063677762048 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17104896 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚322 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚461332211830798 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚322 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚56576831 K>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59053056 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚405265143090176 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚457264675880960 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚ 118991812 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 5368709120 IBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚60 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚62419274 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚41948160 H@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚665 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 4067477562410 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚322 G@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚60 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚464063677762048 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 5494538240 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 4552982528 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚429 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚14142756 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 trainer/global_step‚16849 D5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5264 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 4654602240 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 4075961185792 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚56572538 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚14142696 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 125829120 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚62419274 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚53005838 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚ 118996107 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚459987674628096 B9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚5264 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚56576831 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚38863080 G@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚62 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 122306526 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 _step‚1388 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚41948160 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚4293 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚61478912 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521206 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚322 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚4295 NADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚59053056 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 4075961185792 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 845178880 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚400954565951488 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 5494538240 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 5368709120 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚53005776 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚459987657523200 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 _runtime‚4478.611199629ì »é¢2Ä'- _wandbruntime‚4478  _runtime‚4478`.buÅ'ï &loss/total_loss‚1.8996653368524872 epoch‚42 trainer/global_step‚16849 % _timestamp‚1.7866631300819294e+09 _runtime‚4478.611758413 _step‚1389 0loss/FRt1/assignment_loss‚0.5119496554059851 0loss/FRt2/assignment_loss‚0.4127646316329641 0loss/FRt1/detection_loss‚0.49001759567536085 .loss/FRt2/detection_loss‚0.484933454138177ÂÈzÆ'ô _runtime‚4478.611758413 _step‚1389 0loss/FRt1/assignment_loss‚0.5119496554059851 0loss/FRt2/assignment_loss‚0.4127646316329641 0loss/FRt1/detection_loss‚0.49001759567536085 .loss/FRt2/detection_loss‚0.484933454138177 &loss/total_loss‚1.8996653368524872 epoch‚42 trainer/global_step‚16849 % _timestamp‚1.7866631300819294e+09í ¸@˜ ¡Ç'‚(:– à¡ùÓÌçžá gpu.0.gpu‚64.0 gpu.0.memory‚12, gpu.0.memoryAllocated‚7.0313262939453125) gpu.0.memoryAllocatedBytes‚ 6039863296 gpu.0.temp‚37.0 gpu.0.powerWatts‚200.978' gpu.0.enforcedPowerLimitWatts‚500.0) gpu.0.powerPercent‚40.195600000000006 gpu.0.smClock‚1410 gpu.0.memoryClock‚1593$ gpu.0.correctedMemoryErrors‚1955# gpu.0.uncorrectedMemoryErrors‚0ò<`¬–È'‚(:‹ à¡ùÓà±–ä$ memory_percent‚68.61357688903809 cpu‚27.246818521444446 disk.sda.out‚ 83.234375 disk.sda1.out‚0 disk.drbd2165.out‚0 proc.cpu.threads‚17 network.recv‚ 8.429188e+06* disk./.usagePercent‚32.728031338107556 disk.sda2.in‚0& disk.nvme0n1.out‚61373.25048828125 disk.dm-1.out‚ 83.234375 disk.drbd2165.in‚0 disk.dm-0.in‚44183.52734375 disk.sda3.in‚ 41893.640625 network.sent‚ 2.1958689e+07$ disk.nvme0c0n1.in‚44183.52734375( disk.nvme0c0n1.out‚61373.25048828125 disk.sda.in‚ 41893.640625 disk.sda3.out‚ 83.234375( proc.memory.availableMB‚ 5142.3515625# proc.memory.rssMB‚ 2354.24609375) proc.memory.percent‚14.36917781829834$ disk./.usageGB‚487.6713180541992 disk.sda1.in‚0 disk.sda2.out‚0" disk.nvme0n1.in‚44183.52734375 disk.dm-1.in‚ 41893.640625# disk.dm-0.out‚61373.25048828125WiÂ¥2É'- _wandbruntime‚4491  _runtime‚4491¢ÌnÈÊ' _step‚1390 & lr-AdamW/pg1‚9.075336888419319e-05 & lr-AdamW/pg2‚9.075336888419319e-05 trainer/global_step‚16899 $ _timestamp‚1.786663142596688e+09 _runtime‚4491.126799804óož‚ÍË'Ç & lr-AdamW/pg1‚9.075336888419319e-05 & lr-AdamW/pg2‚9.075336888419319e-05 trainer/global_step‚16899 $ _timestamp‚1.786663142596688e+09 _runtime‚4491.126799804 _step‚1390î Ëðiz2Ì'- _wandbruntime‚4491  _runtime‚4491KþdÐÅ'Í'¿O JDeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4289 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚429 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚462626588564630 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚322 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚56734353 G>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4876 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚461277469179904 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚262 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚209 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 PCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚59049984 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 5368709120 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚262 IBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚78 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚406406503692800 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 4322616474112 trainer/global_step‚16899 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚ 119327724 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚41945088 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚38971127 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚406406428525056 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚62593371 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚461277452075008 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 845178880 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 5368709120 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚38971125 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚56738640 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 122905600 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚465364828007936 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 831258624 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚458547779764224 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚262 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚322 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 4517085184 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 5494538240 BDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚461277469179904 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 4087358828032 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 122905600 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚322 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 4654602240 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚56738640 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚14183274 UHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚75167744 PCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚58387380 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 125829120 _runtime‚4491.130874717 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚462626646952010 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚465364768957952 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚465364768957952 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 122306526 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 5494538240 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 4087358828032 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚41945088 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚14183198 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 4078808800406 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚53154401 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 4552982528 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 4087316882944 M@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚59049984 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17104896 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 4078850148426 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 5494538240 JBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚665 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 % _timestamp‚1.7866631425977187e+09 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚4287 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚223 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚54523904 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 5368709120 KDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚60 _step‚1391 C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚4289 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 PIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚76 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 4628521206 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚461277452075008 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚458547796803584 @7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚5264 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚62593373 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚61478912 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚56734353 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚402083866574848 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚41348020 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 4087316882944 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17104896 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 4552982528 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚53154323 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚62593371 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚4287 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 4322561950208 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0¯ƒþ Î'ÄO TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚38971125 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚56738640 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 122905600 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚465364828007936 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 831258624 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚458547779764224 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚262 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚322 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 4517085184 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 5494538240 BDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚461277469179904 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 4087358828032 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 122905600 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚322 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 4654602240 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚56738640 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚14183274 UHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚75167744 PCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚58387380 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 125829120 _runtime‚4491.130874717 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚462626646952010 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚465364768957952 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚465364768957952 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 122306526 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 5494538240 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 4087358828032 TGDeviŠòeþÌceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚41945088 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚14183198 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 4078808800406 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚53154401 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 4552982528 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 4087316882944 M@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚59049984 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17104896 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 4078850148426 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 5494538240 JBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚665 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 % _timestamp‚1.7866631425977187e+09 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚4287 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚223 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚54523904 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 5368709120 KDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚60 _step‚1391 C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚4289 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 PIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚76 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 4628521206 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚461277452075008 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚458547796803584 @7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚5264 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚62593373 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚61478912 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚56734353 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚402083866574848 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚41348020 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 4087316882944 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17104896 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 4552982528 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚53154323 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚62593371 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚4287 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 4322561950208 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 JDeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4289 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚429 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚462626588564630 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚322 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚56734353 G>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4876 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚461277469179904 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚262 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚209 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 PCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚59049984 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 5368709120 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚262 IBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚78 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚406406503692800 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 4322616474112 trainer/global_step‚16899 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚ 119327724 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚41945088 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚38971127 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚406406428525056 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚62593371 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚461277452075008 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 845178880 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 5368709120ï ¿‡ð02Ï'- _wandbruntime‚4491  _runtime‚4491ÁÛÚÅÎ&Ð'ÈM SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 122905600 I@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚4876 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 4517085184 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚322 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 4087589899776 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚262 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 125829120 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚62596926 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚56737640 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚461303791517696 NGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚78 EDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚53157351 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 845178880 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚262 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚465391381417472 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚ 119334564 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 831258624 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚462653004930778 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚38973330 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚322 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚462653063318206 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚ 119338859 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚4295 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 4087547951616 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 119338859 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚62596924 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚50326528 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 4322809766400 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚56741933 B9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚5264 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚61478912 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521206 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚458573982597120 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚62596924 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17104896 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚41948160 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚14184021 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 4087547951616 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 4079080721086 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚322 _runtime‚4491.299288868 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚56741933 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚402106913947648 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 5494538240 D=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚60 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚ 119334564 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚465391381417472 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 122905600 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 4552982528 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚14184099 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 4079039373018 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 122306526 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 4087589899776 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚461303791517696 BDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚461303774412800 G@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚80 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚322 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚38973332 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚406429723714048 trainer/global_step‚16899 _step‚1392 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 H@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚665 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚4293 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚41948160 K>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59053056 DDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 5368709120 >5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5264 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚465391322364416 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654602240 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 4654602240 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚53157431 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚406429794684416 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚402106934591488 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 NADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚58387428 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚461303774412800 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 5494538240 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 5494538240 P6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚322 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚462653063318206 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚ 119338859 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚4295 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 4087547951616 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 119338859 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚62596924 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚50326528 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 4322809766400 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚56741933 B9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚5264 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚61478912 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521206 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚458573982597120 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚62596924 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17104896 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚41948160 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚14184021 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 4087547951616 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 4079080721086 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚322 _runtime‚4491.299288868 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚56741933 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚402106913947648 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 5494538240 D=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚60 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚ 119334564 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚465391381417472 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 122905600 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 4552982528 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚14184099 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 4079039373018 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 122306526 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 4087589899776 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚461303791517696 BDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚461303774412800 G@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚80 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚322 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚38973332 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚406429723714048 trainer/global_step‚16899 _step‚1392 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 H@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚665 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚4293 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚41948160 K>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59053056 DDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 5368709120 >5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5264 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚465391322364416 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654602240 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 4654602240 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚53157431 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚406429794684416 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚402106934591488 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 NADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚58387428 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚461303774412800 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 5494538240 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 5494538240 PDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚53157351 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 845178880 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚262 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚465391381417472 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚ 119334564 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 831258624 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚462653004930778ð @¯Ö2Ò'- _wandbruntime‚4491  _runtime‚4491΢¬¨vÓ'ð 1loss/FRt2/assignment_loss‚0.38213869694937275 0loss/FRt1/detection_loss‚0.46635987019113306 %loss/total_loss‚1.752054948060669 epoch‚42 trainer/global_step‚16899 _runtime‚4491.299793051 0loss/FRt1/assignment_loss‚0.4546183462344947 /loss/FRt2/detection_loss‚0.4489380346856687 % _timestamp‚1.7866631427699318e+09 _step‚1393*¶$J{Ô'õ 1loss/FRt2/assignment_loss‚0.38213869694937275 0loss/FRt1/detection_loss‚0.46635987019113306 %loss/total_loss‚1.752054948060669 epoch‚42 trainer/global_step‚16899 _runtime‚4491.299793051 0loss/FRt1/assignment_loss‚0.4546183462344947 /loss/FRt2/detection_loss‚0.4489380346856687 % _timestamp‚1.7866631427699318e+09 _step‚1393ñ ÓÄ]”Õ'‚(:‰ ï¡ùÓ‹®ãà gpu.0.gpu‚0.0 gpu.0.memory‚0, gpu.0.memoryAllocated‚7.0313262939453125) gpu.0.memoryAllocatedBytes‚ 6039863296 gpu.0.temp‚37.0 gpu.0.powerWatts‚209.043' gpu.0.enforcedPowerLimitWatts‚500.0 gpu.0.powerPercent‚41.8086 gpu.0.smClock‚1410 gpu.0.memoryClock‚1593$ gpu.0.correctedMemoryErrors‚1955# gpu.0.uncorrectedMemoryErrors‚0¡¿¨ùÖ'‚(:… ï¡ùÓþ¯îâ$ memory_percent‚68.61224174499512 cpu‚27.24645966738563 disk.dm-1.out‚ 83.3984375 disk.drbd2165.out‚0 disk.dm-0.in‚44248.26171875 disk.sda.in‚ 41893.640625 disk.sda1.out‚0 disk.sda2.in‚0$ disk.nvme0n1.out‚61421.833984375) proc.memory.percent‚14.36917781829834& disk.nvme0c0n1.out‚61421.833984375 network.sent‚ 2.203094e+07* disk./.usagePercent‚32.728039530348646$ disk./.usageGB‚487.6714401245117 disk.sda3.out‚ 83.3984375 disk.sda.out‚ 83.3984375 disk.sda1.in‚0 disk.sda2.out‚0! disk.dm-0.out‚61421.833984375$ disk.nvme0c0n1.in‚44248.26171875" disk.nvme0n1.in‚44248.26171875( proc.memory.availableMB‚ 5142.5703125# proc.memory.rssMB‚ 2354.24609375 proc.cpu.threads‚17 network.recv‚ 8.45842e+06 disk.dm-1.in‚ 41893.640625 disk.drbd2165.in‚0 disk.sda3.in‚ 41893.640625*hP2×'- _wandbruntime‚4503  _runtime‚4503q6€9ÉØ'à & lr-AdamW/pg2‚8.790848816275416e-05 trainer/global_step‚16949 % _timestamp‚1.7866631552679493e+09 _runtime‚4503.798019939 _step‚1394 & lr-AdamW/pg1‚8.790848816275416e-05oö·åÎÙ'È & lr-AdamW/pg1‚8.790848816275416e-05 & lr-AdamW/pg2‚8.790848816275416e-05 trainer/global_step‚16949 % _timestamp‚1.7866631552679493e+09 _runtime‚4503.798019939 _step‚1394ò ¨ê$.2Ú'- _wandbruntime‚4503  _runtime‚4503%¨(½ Û'¿O JBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚665 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚403236235214848 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 4334895234048 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 5368709120 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 JDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 4098945593856 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚466692472613376 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚14223851 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚459857086480384 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 122905600 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 125829120 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚262 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 4098903648768 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚53305228 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚41348020 IBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚60 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚4287 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17104896 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 122905600 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚56899455 KBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚4876 IBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚63 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚262 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚40757115V"K1092736 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚407571090605056 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚4289 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚39081377 _step‚1395 @7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚5264 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚466692531663360 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚429 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚ 119670476 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 4552982528 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚ 119674765 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚14223790 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚459857069441024 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚322 F?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚60 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚466692472613376 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚322 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 4090411959034 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 5494538240 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚62771023 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚4287 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚262 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚62771021 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚56903742 PCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚58387380 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 5494538240 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚223 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚403236255858688 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 4628521206 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 5368709120 KDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚60 % _timestamp‚1.7866631552689805e+09 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚56899455 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 4654602240 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17104896 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚209 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚20643840 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚39843840 BDeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4289 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 5494538240 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚62771023 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚462593586069504 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 4098945593856 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 125829120 G>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4876 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 845178880 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 trainer/global_step‚16949 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 4517085184 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 4090370611014 M@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚59049984 JDeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4289 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 5494538240 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚62771023 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚462593586069504 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 4098945593856 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 125829120 G>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4876 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 845178880 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 trainer/global_step‚16949 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 4517085184 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 4090370611014 M@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚59049984 JDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 4098945593856 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚466692472613376 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚14223851 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚459857086480384 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 122905600 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 125829120 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚262 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 4098903648768 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚53305228 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚41348020 IBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚60 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚4287 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17104896 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 122905600 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚56899455 KBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚4876 IBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚63 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚262 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚407571151092736 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚407571090605056 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚4289 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚39081377 _step‚1395 @7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚5264 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚466692531663360 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚429 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚ 119670476 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 4552982528 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚ 119674765 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚14223790 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚459857069441024 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚322 F?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚60 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚466692472613376 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚322 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 4090411959034 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 5494538240 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚62771023 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚4287 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚262 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚62771021 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚56903742 PCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚58387380 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 5494538240ó S+N42Ý'- _wandbruntime‚4503  _runtime‚4503õGWÏ&Þ'ÉM >5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5264 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 4552982528 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚56907035 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17104896 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚39083580 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚14224667 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 5368709120 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚462619891302400 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚41948160 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚407594445393920 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 _runtime‚4503.981082145 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚56907035 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚429 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 4099176665600 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚223 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 845178880 SFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚58387456 _step‚1396 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚209 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 NADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚58387428 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚463973856418186 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 4099134717440 G@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚60 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚ 119677316 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚53308249 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚466719085072896 B9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚5264 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 4517085184 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚41348068 % _timestamp‚1.7866631554506838e+09 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚62774576 EDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 119681611 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 5368709120 I@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚4876 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚41948160 NADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚59053056 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚62774576 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚459883255234560 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚4295 K>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59053056 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521206 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 5368709120 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 IBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚60 EDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚53308180 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 125829120 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚322 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚322 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17104896 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 4099176665600 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚466719026019840 BDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 122905600 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 4335142162432 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚4293 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚56902742 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚459883272273920 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 4099134717440 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚262 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚462619908407296 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 4552982528 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 125829120 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚262 trainer/global_step‚16949 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 122905600 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚20643840 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚61478912 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654602240 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚462619908407296 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚ 119677316 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚322 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚262še ß'ÎM ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚459883255234560 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚4295 K>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59053056 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521206 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 5368709120 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 IBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚60 EDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚53308180 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 125829120 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚322 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚322 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17104896 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 4099176665600 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚466719026019840 BDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 122905600 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 4335142162432 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚4293 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚56902742 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚459883272273920 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 4099134717440 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚262 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚462619908407296 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 4552982528 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 125829120 H@DeviceStatsMonitor.on_traùR#Êin_batch_end/segment.large_pool.current‚262 trainer/global_step‚16949 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 122905600 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚20643840 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚61478912 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654602240 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚462619908407296 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚ 119677316 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚322 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚262 >5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5264 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 4552982528 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚56907035 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17104896 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚39083580 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚14224667 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 5368709120 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚462619891302400 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚41948160 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚407594445393920 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 _runtime‚4503.981082145 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚56907035 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚429 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 4099176665600 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚223 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 845178880 SFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚58387456 _step‚1396 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚209 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 NADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚58387428 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚463973856418186 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 4099134717440 G@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚60 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚ 119677316 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚53308249 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚466719085072896 B9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚5264 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 4517085184 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚41348068 % _timestamp‚1.7866631554506838e+09 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚62774576 EDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 119681611 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 5368709120 I@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚4876 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚41948160 NADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚59053056 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚62774576ô ½³82à'- _wandbruntime‚4503  _runtime‚4503â³uá'ï _step‚1397 0loss/FRt1/assignment_loss‚0.4787144186305716 &loss/total_loss‚1.8397295763919277 epoch‚42 trainer/global_step‚16949 _runtime‚4503.981754375 0loss/FRt2/assignment_loss‚0.3797139163050163 /loss/FRt1/detection_loss‚0.5075949419855178 0loss/FRt2/detection_loss‚0.47370629947082205 $ _timestamp‚1.786663155451873e+09ípzâ'ô 0loss/FRt2/assignment_loss‚0.3797139163050163 /loss/FRt1/detection_loss‚0.5075949419855178 0loss/FRt2/detection_loss‚0.47370629947082205 $ _timestamp‚1.786663155451873e+09 _step‚1397 0loss/FRt1/assignment_loss‚0.4787144186305716 &loss/total_loss‚1.8397295763919277 epoch‚42 trainer/global_step‚16949 _runtime‚4503.981754375õ _Él¡ã'‚(:– þ¡ùÓÕ•ùà gpu.0.gpu‚64.0 gpu.0.memory‚12, gpu.0.memoryAllocated‚7.0313262939453125) gpu.0.memoryAllocatedBytes‚ 6039863296 gpu.0.temp‚37.0 gpu.0.powerWatts‚222.476' gpu.0.enforcedPowerLimitWatts‚500.0) gpu.0.powerPercent‚44.495200000000004 gpu.0.smClock‚1410 gpu.0.memoryClock‚1593$ gpu.0.correctedMemoryErrors‚1955# gpu.0.uncorrectedMemoryErrors‚0ùH„—Žä'‚(:ƒ þ¡ùÓߌÎã( disk.nvme0c0n1.out‚61425.57958984375 disk.sda1.in‚0 disk.sda1.out‚0& proc.memory.availableMB‚ 5142.34375) proc.memory.percent‚14.36917781829834 network.sent‚ 2.2103937e+07 network.recv‚ 8.488465e+06 disk.sda2.out‚0 disk.drbd2165.in‚0 disk.dm-0.in‚44293.48046875 disk.sda3.in‚ 41893.640625$ disk.nvme0c0n1.in‚44293.48046875 disk.sda.out‚83.53125$ disk./.usageGB‚487.6713333129883" disk.nvme0n1.in‚44293.48046875 disk.dm-1.out‚83.53125 disk.drbd2165.out‚0 disk.sda3.out‚83.53125 disk.sda.in‚ 41893.640625# memory_percent‚68.6136245727539# proc.memory.rssMB‚ 2354.24609375 disk.sda2.in‚0 disk.dm-1.in‚ 41893.640625 cpu‚27.24635239255618 proc.cpu.threads‚17) disk./.usagePercent‚32.72803236213769& disk.nvme0n1.out‚61425.57958984375# disk.dm-0.out‚61425.57958984375IJ9:2å'- _wandbruntime‚4516  _runtime‚4516Cº“¡Çæ'Á _step‚1398 & lr-AdamW/pg1‚8.510613234190157e-05 & lr-AdamW/pg2‚8.510613234190157e-05 trainer/global_step‚16999 % _timestamp‚1.7866631676904585e+09 _runtime‚ 4516.2229307éI/ŸÌç'Æ _runtime‚ 4516.2229307 _step‚1398 & lr-AdamW/pg1‚8.510613234190157e-05 & lr-AdamW/pg2‚8.510613234190157e-05 trainer/global_step‚16999 % _timestamp‚1.7866631676904585e+09ö FFuD2è'- _wandbruntime‚4516  _runtime‚45165‰ÄRÄ'é'¾O QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚53455868 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚262 F?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚60 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚463909702959104 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚463909685854208 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚41945088 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚57068844 PCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚58387380 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 4110490414592 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 4552982528 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚ 120017517 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 831258624 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 4628521206 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 5494538240 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚62948673 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 5368709120 KDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚60 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚465268291539446 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 4101973769642 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 5368709120 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚322 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 F>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 PCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚59049984 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚4287 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚41945088 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 125829120 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚468020235318784 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 4654602240 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚57064557 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚408735740158976 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚61478912 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚53455803 UHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚64681984 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 4347180342272 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚44038144 JDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4876 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚404388624498688 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 trainer/global_step‚16999 _runtime‚4516.223029026 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚62948671 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚39191627 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚408735804840960 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚461166376157184 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 125829120 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 4517085184 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚39191625 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚463909685854208 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚57064557 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚461166359117824 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚57068844 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 4101932421622 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 4110490414592 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 122905600 KBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚4876 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚14264241 BDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 @7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚5264 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 4110532359680 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 122905600 PIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚63 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 845178880 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚429 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚404388603854848 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚262 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚465268349926826 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚62948671 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚468020176268800 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 5494538240 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 D;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚5264 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 5368709120 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 125829120 C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚4289 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4289 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚468020235318784 IBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚65 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17104896ÀïÍ«É'ê'ÃO PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚57068844 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 4101932421622 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 4110490414592 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 122905600 KBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚4876 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚14264241 BDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 @7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚5264 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 4110532359680 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 122905600 PIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚63 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 845178880 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚429 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚404388603854848 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚262 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚465268349926826 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚62948671 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚468020176268800 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 5494538240 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 D;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚5264 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 5368709120 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 125829120 C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚4289 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4289 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚468020235318784 IBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚65 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17104896 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚53455868 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚262 F?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚60 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚463909702959104 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚463909685854208 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚41945088 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚57068844 PCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚58387380 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 4110490414592 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 4552982528 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚ 120017517 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 831258624 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 4628521206 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 5494538240 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚62948673 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 5368709120 KDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚60 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚465268291539446 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 4101973769642 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 5368709120 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚322 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 F>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 PCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚59049984 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚4287 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚41945088 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 125829120 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚468020235318784 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 4654602240 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚57064557 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚408735740158976 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚61478912 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚53455803 UHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚64681984 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 4347180342272 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚44038144 JDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4876 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚404388624498688 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 trainer/global_step‚16999 _runtime‚4516.223029026 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚62948671 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚39191627 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚408735804840960 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚461166376157184 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 125829120 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 4517085184 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚39191625 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚463909685854208 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚57064557 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚461166359117824 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0÷ 21ì2ë'- _wandbruntime‚4516  _runtime‚4516A¢Îì'ÈM VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚39193832 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚39193830 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚14264946 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚322 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚4293 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚463936025296896 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚62952224 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚57067844 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 845178880 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚322 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚4295 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚41948160 H@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚665 I@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚4876 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 831258624 BDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚53458776 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚209 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚39840768 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 5368709120 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 K>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59053056 NGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚66 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚ 120020068 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚223 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521206 G@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚60 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654602240 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 5494538240 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚468046788728320 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚463936025296896 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 125829120 IBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚60 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚468046729675264 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 4110763431424 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚408759095668736 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 NADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚58387428 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 125829120 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 D5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5264 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 4347423797248 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚404411651227648 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 125829120 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 4552982528 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚62952226 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 4102204342302 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚41348068 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 4552982528 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17104896 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 4110721483264 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚322 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 122306526 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚322 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚ 120024363 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚463936008192000 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚14265012 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚408759035184128 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚461192544911360 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚262 D=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚60 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 4110763431424 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 4110721483264 NADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚59053056 B9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚5264 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚61478912 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚262 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚57072137 G@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚68 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 5494538240 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 4347383956480 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 5368709120 trainer/global_step‚16999 % _timestamp‚1.7866631678520582e+09 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚57067844 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 EDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 120024363 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚ 120020068 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 4517085184 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 4102162994234 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 5368709120 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚468046788728320 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 4654602240 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚20643840 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 PDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 120024363 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚ 120020068 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 4517085184 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 4102162994234 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 5368709120 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚468046788728320 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 4654602240 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚20643840 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 P6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚322 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚4295 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚41948160 H@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚665 I@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚4876 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 831258624 BDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚53458776 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚209 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚39840768 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 5368709120 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 K>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59053056 NGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚66 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚ 120020068 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚223 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521206 G@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚60 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654602240 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 5494538240 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚468046788728320 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚463936025296896 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 125829120 IBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚60 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚468046729675264 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 4110763431424 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚408759095668736 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 NADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚58387428 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 125829120 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 D5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5264 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 4347423797248 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚404411651227648 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 125829120 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 4552982528 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚62952226 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 4102204342302 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚41348068 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 4552982528 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17104896 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 4110721483264 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚322 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 122306526 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚322 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚ 120024363 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚463936008192000 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚14265012 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚408759035184128 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚461192544911360 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚262 D=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚60 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 4110763431424 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 4110721483264 NADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚59053056 B9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚5264 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚61478912 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚262 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚57072137 G@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚68 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 5494538240 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 4347383956480 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 5368709120 trainer/global_step‚16999 % _timestamp‚1.7866631678520582e+09 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚57067844 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 EDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 4122077180416 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 5368709120 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚469347879924224 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 125829120 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚39301875 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 4552982528 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚ 120360269 KBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚4876 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 IBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚77 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚14304381 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚262 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚4287 PCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚58387380 $ _timestamp‚1.786663180685193e+09 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚57229659 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 4122077180416 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚465225802743808 D;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚5264 F?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚60 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚469347879924224 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚262 C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚4289 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚53606256 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚409900469566976 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 IBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚60 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 4552982528 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17104896 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 122905600 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚39301877 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚14304456 UHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚62584832 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 4359434487296 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚63126321 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚466589201414234 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 5494538240 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 _runtime‚4529.218872561 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚405540972494848 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚462475665833984 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚469347938974208 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 4517085184 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚ 120355980 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚465225802743808 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 4113535580250 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚63126323 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚63126321 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚41940992 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 4628521206 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚262 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚57229659 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 125829120 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 _step‚1403 G>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4876 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚209 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚61478912 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 5494538240 JDeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4289 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 4359476428288 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 5368709120 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚322*ßXâö'ÃO C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚4289 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚53606256 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚409900469566976 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 IBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚60 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 4552982528 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17104896 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 122905600 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚39301877 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚14304456 UHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚62584832 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 4359434487296 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚63126321 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚466589201414234 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 5494538240 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 _runtime‚4529.218872561 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚405540972494848 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚462475665833984 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚469347938974208 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 4517085184 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚ 120355980 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚465225802743808 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 4113535580250 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚63126323 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚63126321 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚41940992 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 4628521206 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚262 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚57229659 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 125829120 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 _step‚1403 G>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4876 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚209 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚61478912 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 5494538240 JDeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4289 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 4359476428288 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 5368709120 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚322 PCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚59049984 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 125829120 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚322 M@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚59049984 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚41945088 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚41348020 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚41945088 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚20643840 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚466589143026854 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 @7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚5264 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17104896 JDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 4122077180416 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 5368709120 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚469347879924224 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 125829120 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚39301875 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 4552982528 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚ 120360269 KBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚4876 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 IBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚77 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚14304381 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚262 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚4287 PCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚58387380 $ _timestamp‚1.786663180685193e+09 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚57229659 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 4122077180416 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚465225802743808 D;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚5264 F?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚60 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚469347879924224 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚262û NŠTÛ2÷'- _wandbruntime‚4529  _runtime‚4529DožMÏ&ø'ÉM XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 4122308249088 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚63129876 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚39304080 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚209 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚262 D=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚60 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚63129876 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚405564040511488 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 4359725453824 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚48229376 _step‚1404 PDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚322 EDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚465252125081600 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚20643840 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 5494538240 _runtime‚4529.377054419 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚57232946 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚223 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚14305238 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 845178880 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 120367115 SFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚68873216 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚41348068 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚322 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚63129874 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 4552982528 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚39304082 NGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚82 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521206 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚465252125081600 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚63129874 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚57232946 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚429 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚409923765965312 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 4122350197248 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚57237239 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 5368709120 I@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚4876 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚409923697092096 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 122306526 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 4122350197248 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚466615559393002 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 5368709120 % _timestamp‚1.7866631808467636e+09 DDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 IBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚60 trainer/global_step‚17049 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚ 120362820 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 831258624 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚462501851627520 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚262 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 122905600 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚465252142186496 H@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚665 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚4293 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 4113766152910 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚262 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚41948160 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 4122308249088 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654602240 >5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5264 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17104896 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚405564019867648 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚61478912 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚322 BDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59053056 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚53609236 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚14305156 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 5368709120 E6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚322 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚ 120367115 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚41948160 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0R»[îÔ&ù'ÎM I@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚4876 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚409923697092096 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 122306526 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 4122350197248 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚466615559393002 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 5368709120 % _timestamp‚1.7866631808467636e+09 DDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 IBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚60 trainer/global_step‚17049 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚ 120362820 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 831258624 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚462501851627520 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚262 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 122905600 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚465252142186496 H@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚665 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚4293 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 4113766152910 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚262 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚41948160 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 4122308249088 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654602240 >5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5264 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17104896 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚405564019867648 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚61478912 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚322 BDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59053056 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚53609236 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚14305156 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 5368709120 E6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚322 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚ 120367115 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚41948160 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 4122308249088 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚63129876 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚39304080 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚209 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚262 D=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚60 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚63129876 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚405564040511488 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 4359725453824 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚48229376 _step‚1404 PDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚322 EDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚465252125081600 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚20643840 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 5494538240 _runtime‚4529.377054419 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚57232946 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚223 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚14305238 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 845178880 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 120367115 SFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚68873216 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚41348068 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚322 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚63129874 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 4552982528 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚39304082 NGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚82 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521206 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚465252125081600 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚63129874 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚57232946 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚429 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚409923765965312 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 4122350197248 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚57237239 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 5368709120ü Ì~¥2ú'- _wandbruntime‚4529  _runtime‚4529n¿§;tû'î 0loss/FRt1/assignment_loss‚0.5607179896804008 /loss/FRt1/detection_loss‚0.5138793995181575 /loss/FRt2/detection_loss‚0.4936807450553911 &loss/total_loss‚1.9742035084130514 epoch‚42 % _timestamp‚1.7866631808477187e+09 0loss/FRt2/assignment_loss‚0.4059253741591019 trainer/global_step‚17049 _runtime‚ 4529.37751512 _step‚1405¹]UMyü'ó 0loss/FRt1/assignment_loss‚0.5607179896804008 /loss/FRt1/detection_loss‚0.5138793995181575 /loss/FRt2/detection_loss‚0.4936807450553911 &loss/total_loss‚1.9742035084130514 epoch‚42 % _timestamp‚1.7866631808477187e+09 0loss/FRt2/assignment_loss‚0.4059253741591019 trainer/global_step‚17049 _runtime‚ 4529.37751512 _step‚1405ý ¿^4–ý'‚(:‹ ¢ùÓ€–ƒá gpu.0.gpu‚64.0 gpu.0.memory‚12, gpu.0.memoryAllocated‚7.0313262939453125) gpu.0.memoryAllocatedBytes‚ 6039863296 gpu.0.temp‚37.0 gpu.0.powerWatts‚217.194' gpu.0.enforcedPowerLimitWatts‚500.0 gpu.0.powerPercent‚43.4388 gpu.0.smClock‚1410 gpu.0.memoryClock‚1593$ gpu.0.correctedMemoryErrors‚1955# gpu.0.uncorrectedMemoryErrors‚0vFw¡~þ'‚(:ó ¢ùÓ£Õ³ã network.recv‚ 8.521176e+06$ disk./.usageGB‚487.6702194213867 disk.sda1.in‚0 disk.sda2.in‚0! disk.nvme0n1.in‚ 50181.9453125 disk.dm-0.in‚ 50181.9453125$ memory_percent‚68.61307621002197# proc.memory.rssMB‚ 2354.24609375 disk.sda3.in‚ 41893.640625 disk.sda.out‚ 83.890625 disk.drbd2165.out‚0 disk.dm-0.out‚ 61472.3125) proc.memory.availableMB‚ 5142.43359375) proc.memory.percent‚14.36917781829834 cpu‚27.245819356817492) disk./.usagePercent‚32.72795760793769! disk.nvme0c0n1.out‚ 61472.3125 disk.sda1.out‚0 disk.sda2.out‚0 disk.nvme0n1.out‚ 61472.3125 disk.dm-1.out‚ 83.890625 network.sent‚ 2.2179056e+07 disk.sda3.out‚ 83.890625# disk.nvme0c0n1.in‚ 50181.9453125 disk.sda.in‚ 41893.640625 disk.dm-1.in‚ 41893.640625 disk.drbd2165.in‚0 proc.cpu.threads‚17ÔI2ÿ'- _wandbruntime‚4541  _runtime‚4541‹¸lÉ€(à _step‚1406 & lr-AdamW/pg1‚7.962971261941135e-05 & lr-AdamW/pg2‚7.962971261941135e-05 trainer/global_step‚17099 % _timestamp‚1.7866631932159858e+09 _runtime‚4541.746022053Yj54Î(È trainer/global_step‚17099 % _timestamp‚1.7866631932159858e+09 _runtime‚4541.746022053 _step‚1406 & lr-AdamW/pg1‚7.962971261941135e-05 & lr-AdamW/pg2‚7.962971261941135e-05þ V®Ì2‚(- _wandbruntime‚4541  _runtime‚4541[£ÄhÒƒ(¿O N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚ 120703021 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 4628521206 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 4517085184 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 F>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚57394761 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 4552982528 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚406693361778688 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 125829120 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚411065120366080 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚463784955510784 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚39412125 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚63303971 G>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4876 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚470675642629632 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚467909994514262 KDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚60 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚429 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚63303971 KBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚4876 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 845178880 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚41348020 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚4289 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 4125097390858 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚322 % _timestamp‚1.7866631932169495e+09 JDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 M@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚59049984 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚470675583579648 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚466541919633408 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 122905600 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4289 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚223 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚466541919633408 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 122905600 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚53757389 F?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚60 _step‚1407 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17104896 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚39412127 @7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚5264 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 4654602240 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚466541936738304 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚41945088 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 4125056042838 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚470675642629632 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚ 120698732 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚209 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚322 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 4133663946240 UHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚66779136 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚53757320 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 4371758587392 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 4371712452096 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 5368709120 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚262 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚470675583579648 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17104896 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚14345195 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚20643840 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 5368709120 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚322 PIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚67 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 122306526 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚262 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚4287 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚463784938471424 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 5494538240 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 5494538240 JBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚665 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 JDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 M@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚59049984 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚470675583579648 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚466541919633408 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 122905600 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4289 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚223 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚466541919633408 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 122905600 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚53757389 F?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚60 _step‚1407 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17104896 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚39412127 @7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚5264 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 4654602240 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚466541936738304 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚41945088 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 4125056042838 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚470675642629632 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚ 120698732 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚209 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚322 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 4133663946240 UHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚66779136 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚53757320 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 4371758587392 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 4371712452096 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 5368709120 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚262 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚470675583579648 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17104896 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚14345195 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚20643840 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 5368709120 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚322 PIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚67 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 122306526 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚262 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚4287 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚463784938471424 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 5494538240 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 5494538240 JBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚665 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 JDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚57394761 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 4552982528 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚406693361778688 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 125829120 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚411065120366080 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚463784955510784 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚39412125 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚63303971 G>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4876 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚470675642629632 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚467909994514262 KDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚60 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚429 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚63303971 KBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚4876 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 845178880 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚41348020 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚4289 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 4125097390858 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚322 % _timestamp‚1.7866631932169495e+09 JDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚53760327 K>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59053056 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚41948160 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 4371955189248 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚466568241971200 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 120709867 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚223 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚406716409151488 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚63307524 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 4133895014912 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 4552982528 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 122905600 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚209 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚406716388507648 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚467936410880410 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚463811141304320 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 5494538240 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 125829120 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚262 G@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚60 trainer/global_step‚17099 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 125829120 _runtime‚4541.926939747 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 4654602240 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 5368709120 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 125829120 D6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚322 BDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 4517085184 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 4125327963518 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚41348068 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 122306526 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚322 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚262 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚4293 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 4372003418624 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 D=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚60 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17104896 NGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚69 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚322 G@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚71 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 IBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚60 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚4295 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚ 120705572 NADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚59053056 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 4133895014912 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚48229376 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521206 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 5368709120 >5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5264 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654602240 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 5494538240 H@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚665 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 5368709120 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚57398048 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17104896 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚463811124264960 B9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚5264 _step‚1408 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚ 120709867 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚4293 I@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚4876 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚467936469267838 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 5494538240 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚63307526 E6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚322 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚39414330 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 SFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚68873216 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 831258624 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚262 % _timestamp‚1.7866631933964617e+09 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 122905600 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚470702196039168 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚63307524 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚53760398 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚429 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0ù   ‡(ÎM PDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 4517085184 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 4125327963518 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚41348068 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 122306526 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚322 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚262 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚4293 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 4372003418624 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 D=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚60 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17104896 NGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚69 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚322 G@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚71 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 IBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚60 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚4295 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚ 120705572 NADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚59053056 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 4133895014912 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚48229376 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521206 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 5368709120 >5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5264 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654602240 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 5494538240 H@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚665 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 5368709120 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚57398048 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17104896 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚463811124264960 B9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚5264 _step‚1408 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚ 120709867 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚4293 I@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚4876 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚467936469267838 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 5494538240 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚63307526 E6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚322 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚39414330 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 SFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚68873216 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 831258624 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚262 % _timestamp‚1.7866631933964617e+09 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 122905600 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚470702196039168 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚63307524 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚53760398 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚429 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 4552982528 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 4133936963072 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 4133936963072 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚ 120705572 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚411088343696896 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚20643840 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚57402341 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚14345997 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚411088412570112 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚466568241971200 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚63307526 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚57402341 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚53760327 K>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59053056 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚41948160 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 4371955189248 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚466568241971200 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 120709867 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚223 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚406716409151488 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚63307524 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 4133895014912 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 4552982528 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 122905600 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚209 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚406716388507648 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚467936410880410 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚463811141304320 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 5494538240 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 125829120 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚262 G@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚60 trainer/global_step‚17099 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 125829120 _runtime‚4541.926939747 JBÝS<µ;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 4654602240 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 5368709120 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 125829120 D6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚322 B˜Œ(‚(: œ¢ùÓù–Íä network.recv‚ 8.551674e+06" disk.nvme0c0n1.in‚ 50194.265625( disk.nvme0c0n1.out‚61477.50927734375& disk.nvme0n1.out‚61477.50927734375 disk.drbd2165.in‚0# disk.dm-0.out‚61477.50927734375$ memory_percent‚68.61348152160645) proc.memory.percent‚14.36917781829834 network.sent‚ 2.2273061e+07$ disk./.usageGB‚487.6710777282715 disk.sda1.in‚0 disk.sda2.in‚0 disk.nvme0n1.in‚ 50194.265625( proc.memory.availableMB‚ 5142.3671875# proc.memory.rssMB‚ 2354.24609375 cpu‚27.245600014535736) disk./.usagePercent‚32.72801520963289 disk.dm-1.in‚41950.08203125 disk.dm-1.out‚ 84.2734375 disk.dm-0.in‚ 50194.265625 proc.cpu.threads‚17 disk.sda3.in‚41950.08203125 disk.sda3.out‚ 84.2734375 disk.sda.in‚41950.08203125 disk.sda.out‚ 84.2734375 disk.sda1.out‚0 disk.sda2.out‚0 disk.drbd2165.out‚0ü*ª2(- _wandbruntime‚4554  _runtime‚4554Q;èAÈŽ( & lr-AdamW/pg1‚7.695600045797876e-05 & lr-AdamW/pg2‚7.695600045797876e-05 trainer/global_step‚17149 $ _timestamp‚1.786663205943438e+09 _runtime‚4554.473444301 _step‚1410Æ“ Í(Ç _step‚1410 & lr-AdamW/pg1‚7.695600045797876e-05 & lr-AdamW/pg2‚7.695600045797876e-05 trainer/global_step‚17149 $ _timestamp‚1.786663205943438e+09 _runtime‚4554.473444301‚ ñL2(- _wandbruntime‚4554  _runtime‚4554àÇÿ)Å'‘(¿O PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚57564150 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 4145250712064 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚53907952 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 845178880 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 5368709120 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 125829120 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚322 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 G>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4876 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 122905600 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚467858036523008 @7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚5264 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚14385575 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 5494538240 JBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚665 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 F?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚60 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 4654602240 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚467858053627904 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚63481623 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚209 BDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 _step‚1411 C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚4289 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚469230904389050 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚39522375 JDeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4289 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚57559863 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚407845709774848 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 4628521206 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚57564150 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 4145292657152 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚322 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚262 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 4552982528 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 PIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚70 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚472003287235072 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚57559863 M@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚59049984 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 4552982528 PCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚59049984 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚429 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚14385505 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 5494538240 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚262 _runtime‚4554.477872825 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚ 121041484 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚4287 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚465094245187584 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚465094228148224 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 4517085184 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 4145250712064 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 831258624 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 4136617853446 trainer/global_step‚17149 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚4287 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚53907880 UHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚70973440 PCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚58387380 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚63481623 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 122905600 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚412229692438528 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚61478912 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 5494538240 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 5368709120 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚472003346285056 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 4145292657152 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17104896 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚322 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 4654602240 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚39522377 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 4384032993280 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 4136659201466 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 122306526 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚407845730418688 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚41348020 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17104896 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚262 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚63481621 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚467858053627904 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 125829120 % _timestamp‚1.7866632059443672e+09 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚ 121045773 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚412229763411968 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚50329600 IBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚60 F>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚223 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 JDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚223 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 JDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4876 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 122905600 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚467858036523008 @7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚5264 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚14385575 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 5494538240 JBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚665 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 F?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚60 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 4654602240 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚467858053627904 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚63481623 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚209 BDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 _step‚1411 C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚4289 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚469230904389050 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚39522375 JDeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4289 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚57559863 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚407845709774848 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 4628521206 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚57564150 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 4145292657152 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚322 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚262 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 4552982528 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 PIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚70 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚472003287235072 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚57559863 M@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚59049984 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 4552982528 PCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚59049984 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚429 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚14385505 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 5494538240 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚262 _runtime‚4554.477872825 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚ 121041484 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚4287 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚465094245187584 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚465094228148224 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 4517085184 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 4145250712064 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 831258624 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 4136617853446 trainer/global_step‚17149 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚4287 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚53907880 UHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚70973440 PCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚58387380 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚63481623 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 122905600 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚412229692438528 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚61478912 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 5494538240 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 5368709120 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚472003346285056 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 4145292657152 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17104896 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚322ƒ wuh2“(- _wandbruntime‚4554  _runtime‚4554NO3â”(ÉM REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚57567443 B6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚322 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚4293 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 4145523728896 DDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654602240 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚469257320755246 G@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚60 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚41948160 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 121052619 EDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521206 E8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 trainer/global_step‚17149 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚472029899694592 K>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59053056 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚57567443 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 122905600 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚63485176 G@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚61 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚465120413941760 >5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5264 PDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17104896 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚209 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 4136889774126 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 5368709120 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚63485174 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚14386334 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚407868777791488 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 4145523728896 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚57563150 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚407868757147648 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚61478912 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 5494538240 I@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚4876 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚53910855 % _timestamp‚1.7866632061063628e+09 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚467884358860800 NGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚59 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚14386275 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚322 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚4295 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚472029840641536 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚429 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚39524580 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚20643840 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 SFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚66776064 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 122306526 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 5494538240 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 4517085184 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚ 121048324 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 NADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚58387428 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚322 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 4145481780736 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 5368709120 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚262 4¡-ïí.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚41348068 NADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚59053056 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 4552982528 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚412252985563136 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 4384274547712 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚472029899694592 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 122905600 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 5368709120 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚ 121052619 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚63485176 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚63485174 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚223 IBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚60 _runtime‚4554.636614789 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚412253052339200 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 4552982528 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚53910916 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚467884375965696 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17104896 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0‘—ÖÔ&•(ÎM L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 121052619 EDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521206 E8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 trainer/global_step‚17149 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚472029899694592 K>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59053056 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚57567443 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 122905600 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚63485176 G@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚61 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚465120413941760 >5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5264 PDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17104896 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚209 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 4136889774126 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 5368709120 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚63485174 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚14386334 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚407868777791488 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 4145523728896 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚57563150 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚407868757147648 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚61478912 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 5494538240 I@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚4876 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚53910855 % _timestamp‚1.7866632061063628e+09 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚467884358860800 NGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚59 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚14386275 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚322 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚4295 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚472029840641536 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚429 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚39524580 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚20643840 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 SFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚66776064 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 122306526 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 5494538240 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 4517085184 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚ 121048324 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 NADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚58387428 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚322 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 4145481780736 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 5368709120 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚262 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚41348068 NADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚59053056 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 4552982528 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚412252985563136 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 4384274547712 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚472029899694592 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 122905600 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 5368709120 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚ 121052619 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚63485176 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚63485174 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚223 IBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚60 _runtime‚4554.636614789 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚412253052339200 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 4552982528 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚53910916 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚467884375965696 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17104896 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚57567443 B6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚322 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚4293 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 4145523728896 DDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654602240 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚469257320755246 G@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚60 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚41948160„ ëìÃÙ2–(- _wandbruntime‚4554  _runtime‚4554<ö„v—(ð 1loss/FRt1/assignment_loss‚0.47725642862561446 1loss/FRt2/assignment_loss‚0.39990188565282775 epoch‚42 trainer/global_step‚17149 _runtime‚4554.637581227 0loss/FRt1/detection_loss‚0.48515059984546494 /loss/FRt2/detection_loss‚0.4562148588801269 %loss/total_loss‚1.818523773004034 $ _timestamp‚1.786663206107804e+09 _step‚1413¶ëQ'{˜(õ _step‚1413 1loss/FRt1/assignment_loss‚0.47725642862561446 1loss/FRt2/assignment_loss‚0.39990188565282775 epoch‚42 trainer/global_step‚17149 _runtime‚4554.637581227 0loss/FRt1/detection_loss‚0.48515059984546494 /loss/FRt2/detection_loss‚0.4562148588801269 %loss/total_loss‚1.818523773004034 $ _timestamp‚1.786663206107804e+09… ³TÙ—–™(‚(:‹ «¢ùÓÄÑÖâ gpu.0.gpu‚52.0 gpu.0.memory‚10, gpu.0.memoryAllocated‚7.0313262939453125) gpu.0.memoryAllocatedBytes‚ 6039863296 gpu.0.temp‚37.0 gpu.0.powerWatts‚207.451' gpu.0.enforcedPowerLimitWatts‚500.0 gpu.0.powerPercent‚41.4902 gpu.0.smClock‚1410 gpu.0.memoryClock‚1593$ gpu.0.correctedMemoryErrors‚1955# gpu.0.uncorrectedMemoryErrors‚0ïÊ …š(‚(:ú «¢ùÓÔØáã$ memory_percent‚68.61557960510254( proc.memory.availableMB‚ 5142.0234375% disk./.usageGB‚487.67101669311523 disk.sda3.out‚ 84.5859375% disk.nvme0c0n1.out‚61525.73046875 disk.sda1.in‚0# disk.nvme0n1.out‚61525.73046875 disk.dm-1.in‚42523.43359375 disk.drbd2165.out‚0 disk.dm-0.in‚50255.25 network.recv‚ 8.579599e+06 disk.sda3.in‚42523.43359375 disk.sda.in‚42523.43359375 disk.sda2.out‚0 disk.drbd2165.in‚0# proc.memory.rssMB‚ 2354.24609375* disk./.usagePercent‚32.728011113512345 disk.nvme0c0n1.in‚50255.25 disk.sda2.in‚0 disk.nvme0n1.in‚50255.25 disk.dm-0.out‚61525.73046875) proc.memory.percent‚14.36917781829834 cpu‚27.245243985994293 proc.cpu.threads‚17 network.sent‚ 2.2343738e+07 disk.sda.out‚ 84.5859375 disk.sda1.out‚0 disk.dm-1.out‚ 84.5859375éÞ|Ó2›(- _wandbruntime‚4566  _runtime‚4566º‹™ìÉœ(à % _timestamp‚1.7866632184463055e+09 _runtime‚4566.976323109 _step‚1414 & lr-AdamW/pg1‚7.432551665495163e-05 & lr-AdamW/pg2‚7.432551665495163e-05 trainer/global_step‚17199¢<‰Î(È _step‚1414 & lr-AdamW/pg1‚7.432551665495163e-05 & lr-AdamW/pg2‚7.432551665495163e-05 trainer/global_step‚17199 % _timestamp‚1.7866632184463055e+09 _runtime‚4566.976323109† uG× 2ž(- _wandbruntime‚4566  _runtime‚4566r0õMÅ'Ÿ(¿O TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚14426527 _runtime‚4566.980869666 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚54059225 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚54059152 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚57724965 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4289 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚413394410638336 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚473331049940480 M@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚59049984 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚4287 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 122905600 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 4517085184 D;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚5264 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚413394348053504 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 G>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4876 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 122905600 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚469174153412608 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 4156837477888 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚408998078414848 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 125829120 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚262 PIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚71 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 4396269638656 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚466403534864384 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚466403517825024 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 4148179664054 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚63659271 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 831258624 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 5368709120 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 125829120 IBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚60 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚63659271 IBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚73 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 5494538240 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 5368709120 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 5494538240 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 125829120 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚322 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚4289 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚41945088 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 JDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚4287 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚41348020 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚57729252 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 4552982528 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 4156879422976 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚63659273 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚14426598 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 845178880 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚322 PCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚59049984 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 4156837477888 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚322 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚ 121384236 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 4156879422976 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚57729252 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚57724965 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚408998099058688 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 4396311579648 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚322 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 5368709120 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚63659273 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 4654602240 JBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚665 KBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚4876 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚209 UHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚62584832 BDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚473330990890496 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 4552982528 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚41945088 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 122306526 KDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚60 F?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚60 _step‚1415 @7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚5264 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚262 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚ 121388525 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚469174170517504 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚473331049940480 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17104896 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚429 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚470551755876458 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚469174170517504 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚469174153412608 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚39632625 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚41940992 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚470551697489078X'dæ (ÄO G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4289 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚413394410638336 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚473331049940480 M@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚59049984 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚4287 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 122905600 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 4517085184 D;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚5264 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚413394348053504 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 G>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4876 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 122905600 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚469174153412608 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 4156837477888 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚408998078414848 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 125829120 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚262 PIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚71 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 4396269638656 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚466403534864384 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚466403517825024 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 4148179664054 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚63659271 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 831258624 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 5368709120 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 125829120 IBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚60 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚63659271 IBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚73 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 5494538240 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 5368709120 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 5494538240 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 125829120 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚322 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚4289 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚41945088 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 JDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚4287 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚41348020 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚57729252 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 4552982528 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 4156879422976 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚63659273 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚14426598 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 845178880 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚322 PCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚59049984 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 4156837477888 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚322 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚ 121384236 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 4156879422976 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚57729252 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚57724965 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚408998099058688 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 4396311579648 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚322 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 5368709120 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚63659273 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 4654602240 JBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚665 KBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚4876 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚209 UHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚62584832 BDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚473330990890496 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 4552982528 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚41945088 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 122306526 KDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚60 F?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚60 _step‚1415 @7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚5264 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚262 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚ 121388525 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚469174170517504 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚473331049940480 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17104896 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚429 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚470551755876458 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚469174170517504 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚469174153412608 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚39632625 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚41940992 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚470551697489078 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚14426527 _runtime‚4566.980869666 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚54059225 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚54059152 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚57724965‡ †‚‹×2¡(- _wandbruntime‚4567  _runtime‚4567xí_aÏ&¢(ÉM QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 845178880 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 4396557295616 B6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚322 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚262 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 121395371 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚44035072 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚322 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚262 IBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚60 >5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5264 H@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚665 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 125829120 _runtime‚4567.144338214 _step‚1416 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚4295 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚473357603350016 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚429 D=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚60 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚223 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚469200475750400 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚409021125787648 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521206 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17104896 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚469200492855296 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚41948160 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 4157068546560 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 4517085184 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚262 DDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654602240 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚57728252 SFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚64678912 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 4552982528 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚41948160 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚14427309 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚466429720657920 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚ 121395371 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚ 121391076 trainer/global_step‚17199 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 122905600 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚ 121391076 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 NGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚75 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 125829120 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 4157110494720 G@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚77 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 4654602240 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚39634830 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚41348068 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚469200475750400 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚57732545 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚466429703618560 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚473357603350016 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 5494538240 % _timestamp‚1.7866632186140418e+09 NADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚59053056 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚39634832 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 5368709120 E6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚322 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 G@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚60 PDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59053056 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚409021146431488 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚20643840 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 831258624 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 122306526 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚57732545 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 122905600 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 4552982528 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚63662824 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚209 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚413417639048192 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 5494538240 EDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 5368709120 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚63662826 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚57728252 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚54062139 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚413417703727104Te~Ô&£(ÎM XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 831258624 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 122306526 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚57732545 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 122905600 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 4552982528 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚63662824 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚209 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚413417639048192 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 5494538240 EDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 5368709120 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚63662826 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚57728252 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚54062139 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚413417703727104 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 845178880 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 4396557295616 B6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚322 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚262 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 121395371 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚44035072 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚322 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚262 IBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚60 >5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5264 H@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚665 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 125829120 _runtime‚4567.144338214 _step‚1416 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚4295 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚473357603350016 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚429 D=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚60 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚223 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚469200475750400 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚409021125787648 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521206 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17104896 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚469200492855296 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚41948160 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 4157068546560 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 4517085184 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚262 DDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654602240 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚57728252 SFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚64678912 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 4552982528 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚41948160 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚14427309 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚466429720657920 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚ 121395371 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚ 121391076 trainer/global_step‚17199 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 122905600 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚ 121391076 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 NGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚75 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 125829120 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 4157110494720 G@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚77 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 4654602240 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚39634830 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚41348068 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚469200475750400 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚57732545 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚466429703618560 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚473357603350016 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 5494538240 % _timestamp‚1.7866632186140418e+09 NADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚59053056 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚39634832 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 5368709120 E6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚322 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 G@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚60 PDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59053056 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚409021146431488 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚20643840ˆ  2¤(- _wandbruntime‚4567  _runtime‚4567< ¦t¥(î trainer/global_step‚17199 % _timestamp‚1.7866632186156104e+09 0loss/FRt2/assignment_loss‚0.4155530086958823 /loss/FRt2/detection_loss‚0.4558439839120421 _runtime‚ 4567.14543064 _step‚1417 0loss/FRt1/assignment_loss‚0.5628862163177428 /loss/FRt1/detection_loss‚0.5090622590122184 &loss/total_loss‚1.9433454679378854 epoch‚42âŠqy¦(ó /loss/FRt2/detection_loss‚0.4558439839120421 _runtime‚ 4567.14543064 _step‚1417 0loss/FRt1/assignment_loss‚0.5628862163177428 /loss/FRt1/detection_loss‚0.5090622590122184 &loss/total_loss‚1.9433454679378854 epoch‚42 trainer/global_step‚17199 % _timestamp‚1.7866632186156104e+09 0loss/FRt2/assignment_loss‚0.4155530086958823‰ !ú˜Ÿ2§(- _wandbruntime‚4572  _runtime‚4572¿•Uר(Ñ 3Purity/1FRt/event_proportion‚0.6297432780265808 /Purity/2FRt/event_purity‚0.4301646649837494 +jet/accuracy_1_of_1‚0.31107205152511597 )jet/accuracy_1_of_2‚0.573611855506897 /particle/accuracy_2_of_2‚0.6833717226982117 -Purity/2FRt/FRt_purity‚0.5644612312316895 /particle/accuracy_1_of_2‚0.9272258281707764 6validation_average_jet_accuracy‚0.3160698413848877 *loss/val_total_loss‚2.5088979742284017 0Purity/*FRt/event_purity‚0.46612271666526794 2Purity/*FRt/event_proportion‚0.669805645942688 Purity/0FRt/event_purity‚1 2Purity/2FRt/event_proportion‚0.040062315762043 particle/accuracy_1_of_1‚1 trainer/global_step‚17199 _runtime‚4572.315402098 +Purity/*FRt/FRt_purity‚0.47930708527565 _step‚1418 -Purity/1FRt/FRt_purity‚0.4683034121990204 % _timestamp‚1.7866632237853498e+09 (particle/accuracy‚0.6405436396598816 'particle/f_score‚0.5559436678886414 (validation_accuracy‚0.21461121737957 /Purity/1FRt/event_purity‚0.4683034121990204 (jet/accuracy_2_of_2‚0.21461121737957 +particle/sensitivity‚0.6340121030807495 +particle/specificity‚0.6441248655319214 epoch‚42 3Purity/0FRt/event_proportion‚0.3301943838596344r±÷Ü©(Ö -Purity/2FRt/FRt_purity‚0.5644612312316895 /particle/accuracy_1_of_2‚0.9272258281707764 6validation_average_jet_accuracy‚0.3160698413848877 *loss/val_total_loss‚2.5088979742284017 0Purity/*FRt/event_purity‚0.46612271666526794 2Purity/*FRt/event_proportion‚0.669805645942688 Purity/0FRt/event_purity‚1 2Purity/2FRt/event_proportion‚0.040062315762043 particle/accuracy_1_of_1‚1 trainer/global_step‚17199 _runtime‚4572.315402098 +Purity/*FRt/FRt_purity‚0.47930708527565 _step‚1418 -Purity/1FRt/FRt_purity‚0.4683034121990204 % _timestamp‚1.7866632237853498e+09 (particle/accuracy‚0.6405436396598816 'particle/f_score‚0.5559436678886414 (validation_accuracy‚0.21461121737957 /Purity/1FRt/event_purity‚0.4683034121990204 (jet/accuracy_2_of_2‚0.21461121737957 +particle/sensitivity‚0.6340121030807495 +particle/specificity‚0.6441248655319214 epoch‚42 3Purity/0FRt/event_proportion‚0.3301943838596344 3Purity/1FRt/event_proportion‚0.6297432780265808 /Purity/2FRt/event_purity‚0.4301646649837494 +jet/accuracy_1_of_1‚0.31107205152511597 )jet/accuracy_1_of_2‚0.573611855506897 /particle/accuracy_2_of_2‚0.6833717226982117Š 3m’–ª(‚(:‹ º¢ùÓ›ƒùà gpu.0.gpu‚58.0 gpu.0.memory‚10, gpu.0.memoryAllocated‚7.0313262939453125) gpu.0.memoryAllocatedBytes‚ 6039863296 gpu.0.temp‚36.0 gpu.0.powerWatts‚195.034' gpu.0.enforcedPowerLimitWatts‚500.0 gpu.0.powerPercent‚39.0068 gpu.0.smClock‚1410 gpu.0.memoryClock‚1593$ gpu.0.correctedMemoryErrors‚1955# gpu.0.uncorrectedMemoryErrors‚0¯¿Øzš«(‚(: º¢ùÓ‹þÕã disk.dm-0.out‚61531.26953125$ memory_percent‚68.61469745635986# proc.memory.rssMB‚ 2354.24609375 cpu‚27.250626959422974 proc.cpu.threads‚17 disk.sda.out‚ 84.77734375 disk.sda2.in‚0 disk.dm-1.out‚ 84.77734375 disk.drbd2165.in‚0 disk.sda3.in‚42523.43359375) proc.memory.availableMB‚ 5142.16796875) proc.memory.percent‚14.36917781829834 network.sent‚ 2.2418494e+07 network.recv‚ 8.611747e+06% disk./.usageGB‚487.67305755615234 disk.sda1.in‚0 disk.dm-1.in‚42523.43359375) disk./.usagePercent‚32.72814807754317$ disk.nvme0c0n1.in‚50333.72265625 disk.sda1.out‚0 disk.sda2.out‚0" disk.nvme0n1.in‚50333.72265625# disk.nvme0n1.out‚61531.26953125 disk.drbd2165.out‚0 disk.sda3.out‚ 84.77734375% disk.nvme0c0n1.out‚61531.26953125 disk.sda.in‚42523.43359375 disk.dm-0.in‚50333.72265625š@KÀ2¬(- _wandbruntime‚4585  _runtime‚4585ÒÈÍÉ­(à & lr-AdamW/pg1‚7.173843016138909e-05 & lr-AdamW/pg2‚7.173843016138909e-05 trainer/global_step‚17249 % _timestamp‚1.7866632367805624e+09 _runtime‚4585.310627095 _step‚1419G§ÞŸÎ®(È & lr-AdamW/pg1‚7.173843016138909e-05 & lr-AdamW/pg2‚7.173843016138909e-05 trainer/global_step‚17249 % _timestamp‚1.7866632367805624e+09 _runtime‚4585.310627095 _step‚1419‹ î7ÒŠ2¯(- _wandbruntime‚4585  _runtime‚4585­•‡¡°(¿O YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚470881207517184 C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚4289 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 122905600 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚57912254 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚209 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 831258624 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 5494538240 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 5494538240 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 KBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚4876 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 5368709120 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚262 _step‚1420 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚41945088 PCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚59049984 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚14475988 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 4163807446763 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 4552982528 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚410449738006528 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 125829120 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17104896 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚54259410 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚472257572827807 PCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚58387380 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 4163848794783 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚41348020 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 5368709120 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 F?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚60 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚223 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 4172491967488 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚475053758534656 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 4628521206 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚262 F>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚57912254 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚475053758534656 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚470881207517184 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚322 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 4172533912576 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 trainer/global_step‚17249 _runtime‚4585.312111082 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚4287 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 845178880 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 125829120 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 4552982528 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚41940992 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 % _timestamp‚1.7866632367816932e+09 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚39783489 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚63897529 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚470881224622080 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚39783491 G>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4876 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 122905600 D;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚5264 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.CÁ¡$current‚4287 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 5494538240 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚61478912 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 122306526 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 5368709120 IBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚60 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚475053699484672 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚475053699484672 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚41945088 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚414862422960640 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚20643840 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚429 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚468093724033024 JDeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4289 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚54259479 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚262|bÊÊ'±(ÄO ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚470881224622080 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚39783491 G>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4876 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 122905600 D;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚5264 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚4287 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 5494538240 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚61478912 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 122306526 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 5368709120 IBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚60 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚475053699484672 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚475053699484672 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚41945088 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚414862422960640 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚20643840 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚429 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚468093724033024 JDeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4289 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚54259479 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚262 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚470881207517184 C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚4289 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 122905600 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚57912254 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚209 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 831258624 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 5494538240 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 5494538240 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 KBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚4876 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 5368709120 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚262 _step‚1420 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚41945088 PCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚59049984 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚14475988 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 4163807446763 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 4552982528 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚410449738006528 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 125829120 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17104896 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚54259410 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚472257572827807 PCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚58387380 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 4163848794783 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚41348020 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 5368709120 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 F?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚60 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚223 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 4172491967488 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚475053758534656 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 4628521206 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚262 F>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚57912254 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚475053758534656 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚470881207517184 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚322 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 4172533912576 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 trainer/global_step‚17249 _runtime‚4585.312111082 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚4287 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 845178880 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 125829120 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 4552982528 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚41940992 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 % _timestamp‚1.7866632367816932e+09 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚39783489 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚63897529Œ !l2²(- _wandbruntime‚4585  _runtime‚4585Ó]CÎ&³(ÈM YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 4172764984320 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚57915541 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚475080252891136 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654602240 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 4172764984320 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚39785694 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 NGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚73 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 4517085184 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 5368709120 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚4293 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚322 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚262 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚14476735 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚322 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 4552982528 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚41948160 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚209 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 4164079367443 E8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 NADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚58387428 D=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚60 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚470907546959872 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚ 121820916 PDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59053056 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚470907546959872 G@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚75 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚472283989194003 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚262 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚ 121816621 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚414885716049408 EDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 H@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚665 _step‚1421 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚470907529854976 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚41948160 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 122905600 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 $ _timestamp‚1.786663236960567e+09 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚63901080 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 4552982528 B9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚5264 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚63901080 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 I@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚4876 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚20643840 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚4293 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 121820916 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚414885647176192 BDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚54262429 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚410472785379328 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 125829120 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 125829120 G@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚60 >5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5264 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚475080311944192 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚14476808 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 5494538240 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚322 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚57919834 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚63901082 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚223 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521206 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 5368709120 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 125829120 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 IBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚60 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚54262504 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 5368709120 _runtime‚4585.490987161 SFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚68873216 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 4164038019375 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚322 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚410472764735488 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 DDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚475080252891136 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654602240 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 4172764984320 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚39785694 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 NGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚73 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 4517085184 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 5368709120 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚4293 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚322 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚262 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚14476735 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚322 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 4552982528 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚41948160 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚209 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 4164079367443 E8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 NADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚58387428 D=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚60 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚470907546959872 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚ 121820916 PDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59053056 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚470907546959872 G@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚75 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚472283989194003 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚262 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚ 121816621 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚414885716049408 EDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 H@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚665 _step‚1421 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚470907529854976 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚41948160 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 122905600 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 $ _timestamp‚1.786663236960567e+09 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚63901080 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 4552982528 B9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚5264 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚63901080 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 I@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚4876 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚20643840 LCDeviceStatsMonitor.on_train_batch_end/Îi±‘äallocation.small_pool.current‚4293 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 121820916 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚414885647176192 BDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚54262429 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚410472785379328 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 125829120 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 125829120 G@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚60 >5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5264 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚475080311944192 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚14476808 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 5494538240 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚322 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚57919834 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚63901082 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚223 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521206 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 5368709120 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 125829120 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 IBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚60 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚54262504 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 5368709120 _runtime‚4585.490987161 êÞKv2µ(- _wandbruntime‚4585  _runtime‚4585^þv¶(ð _runtime‚4585.491624396 0loss/FRt1/assignment_loss‚0.4678334368461661 0loss/FRt2/assignment_loss‚0.3580616570987513 &loss/total_loss‚1.7894066815109764 trainer/global_step‚17249 % _timestamp‚1.7866632369617574e+09 _step‚1422 0loss/FRt1/detection_loss‚0.44770683862457356 /loss/FRt2/detection_loss‚0.5158047489414854 epoch‚43:¬8J{·(õ trainer/global_step‚17249 % _timestamp‚1.7866632369617574e+09 _step‚1422 0loss/FRt1/detection_loss‚0.44770683862457356 /loss/FRt2/detection_loss‚0.5158047489414854 epoch‚43 _runtime‚4585.491624396 0loss/FRt1/assignment_loss‚0.4678334368461661 0loss/FRt2/assignment_loss‚0.3580616570987513 &loss/total_loss‚1.7894066815109764Ž VK°¡¸(‚(:– É¢ùÓÎîŠã gpu.0.gpu‚63.0 gpu.0.memory‚11, gpu.0.memoryAllocated‚7.0313262939453125) gpu.0.memoryAllocatedBytes‚ 6039863296 gpu.0.temp‚37.0 gpu.0.powerWatts‚211.748' gpu.0.enforcedPowerLimitWatts‚500.0) gpu.0.powerPercent‚42.349599999999995 gpu.0.smClock‚1410 gpu.0.memoryClock‚1593$ gpu.0.correctedMemoryErrors‚1955# gpu.0.uncorrectedMemoryErrors‚0£:8n™¹(‚(:Ž É¢ùÓÅïïä# proc.memory.rssMB‚ 2354.24609375 network.sent‚ 2.2494808e+07 network.recv‚ 8.644445e+06) disk./.usagePercent‚32.72816395001029 disk.drbd2165.in‚0 disk.drbd2165.out‚0! disk.dm-0.out‚61579.361328125 disk.sda2.in‚0 disk.sda2.out‚0$ disk./.usageGB‚487.6732940673828& disk.nvme0c0n1.out‚61579.361328125 disk.sda.in‚42523.43359375 disk.sda1.in‚0" disk.nvme0n1.in‚69892.78515625$ memory_percent‚68.61622333526611 cpu‚27.250069756935982 proc.cpu.threads‚17 disk.sda1.out‚0$ disk.nvme0n1.out‚61579.361328125 disk.dm-1.in‚42523.43359375) proc.memory.availableMB‚ 5141.91796875) proc.memory.percent‚14.36917781829834 disk.dm-0.in‚69892.78515625 disk.sda3.in‚42523.43359375 disk.sda3.out‚ 85.2890625$ disk.nvme0c0n1.in‚69892.78515625 disk.sda.out‚ 85.2890625 disk.dm-1.out‚ 85.2890625Àºçƒ2º(- _wandbruntime‚4598  _runtime‚4598&óÑÂÅ»(¿ $ lr-AdamW/pg2‚6.9194907141022e-05 trainer/global_step‚17299 % _timestamp‚1.7866632496764352e+09 _runtime‚4598.206481639 _step‚1423 $ lr-AdamW/pg1‚6.9194907141022e-05‘Mʼ(Ä $ lr-AdamW/pg1‚6.9194907141022e-05 $ lr-AdamW/pg2‚6.9194907141022e-05 trainer/global_step‚17299 % _timestamp‚1.7866632496764352e+09 _runtime‚4598.206481639 _step‚1423 ;{™2½(- _wandbruntime‚4598  _runtime‚4598×Ç=µÅ'¾(¿O G>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4876 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚476381403140096 JDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 % _timestamp‚1.7866632496774375e+09 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚472197341511680 PCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚58387380 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚469403013709824 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 5368709120 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚476381403140096 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚41945088 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚58077356 KBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚4876 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚429 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚322 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 4424957649920 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚473578424315215 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 4175410605391 KDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚60 @7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚5264 F>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 122905600 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚209 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 4628521206 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 122306526 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚469402996670464 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 _step‚1424 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚472197324406784 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 845178880 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 F?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚60 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚472197324406784 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚4287 M@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚59049984 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚476381462190080 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4289 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚ 122152533 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚41945088 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚ 122156822 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚61478912 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 125829120 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 4184120678400 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚14517236 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚411602106646528 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 5368709120 C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚4289 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚476381462190080 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 4184120678400 IBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚60 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚223 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 831258624 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚41348020 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚322 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚54410898óUðÊ'¿(ÄO _runtime‚4598.210887891 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17104896 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚58081643 PIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚77 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 125829120 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚64075179 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚416026995420160 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚48232448 BDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 % _timestamp‚1.7866632496774375e+09 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚472197341511680 PCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚58387380 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚469403013709824 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 5368709120 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚476381403140096 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚41945088 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚58077356 KBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚4876 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚429 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚322 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 4424957649920 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚473578424315215 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 4175410605391 KDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚60 @7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚5264 F>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 122905600 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚209 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 4628521206 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 122306526 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚469402996670464 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 _step‚1424 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚472197324406784 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 845178880 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 F?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚60 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚472197324406784 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚4287 M@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚59049984 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚476381462190080 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4289 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚ 122152533 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚41945088 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚ 122156822 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚61478912 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 125829120 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 4184120678400 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚14517236 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚411602106646528 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 5368709120 C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚4289 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚476381462190080 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 4184120678400 IBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚60 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚223 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 831258624 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚41348020 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚322 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚54410898 G>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4876 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚476381403140096 JDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚54413929 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚411625154019328 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 4517085184 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚14518039 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 4175599829983 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17104896 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚58084936 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚262 NADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚59053056 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚473604840681411 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚64078732 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 122905600 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚476407956546560 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚61478912 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 5368709120 % _timestamp‚1.7866632498506196e+09 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 4175641178051 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 4184351750144 H@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚665 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚411625133375488 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 4552982528 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 4184309801984 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚429 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 4425194649600 trainer/global_step‚17299 DDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 D=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚60 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚4295 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚322 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 122306526 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 125829120 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚ 122159373 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚64078730 NADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚58387428 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚ 122163668 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚472223663849472 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚39895946 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚322 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚4293 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚39895944 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 5494538240 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17104896 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚469429199503360 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚469429182464000 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚262 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚58080643 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 4184351750144 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚58084936 G@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚56 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚209 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 845178880 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 G@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚60 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654602240 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚14517985 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521206 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 122905600 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 122163668 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚416050348668928 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚20643840 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚322 BDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 5368709120 K>DeviceStatsMonitor.on_train_batch_end/active_byte=yÁ€& s.all.current‚59053056 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 4184309801984 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚ 122159373 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚472223646744576 E8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 _step‚1425 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚476408015599616 P5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5264 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 5494538240 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚473604782293983ËužÔ&Â(ÎM TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 4552982528 I@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚4876 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚35646464 SFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚56290304 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚416050292378624 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 EDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 D=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚60 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚4295 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚322 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 122306526 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 125829120 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚ 122159373 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚64078730 NADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚58387428 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚ 122163668 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚472223663849472 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚39895946 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚322 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚4293 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚39895944 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 5494538240 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17104896 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚469429199503360 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚469429182464000 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚262 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚58080643 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 4184351750144 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚58084936 G@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚56 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚209 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 845178880 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 G@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚60 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654602240 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚14517985 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521206 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 122905600 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 122163668 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚416050348668928 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚20643840 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚322 BDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 5368709120 K>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59053056 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 4184309801984 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚ 122159373 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚472223646744576 E8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 _step‚1425 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚476408015599616 P5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5264 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 5494538240 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚473604782293983 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚54413929 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚411625154019328 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 4517085184 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚14518039 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 4175599829983 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17104896 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚58084936 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚262 NADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚59053056 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚473604840681411 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚64078732 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 122905600 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚476407956546560 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚61478912 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 5368709120 % _timestamp‚1.7866632498506196e+09 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 4175641178051 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 4184351750144 H@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚665 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚411625133375488 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 4552982528 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 4184309801984 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚429 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 4425194649600 trainer/global_step‚17299 DöÇœÇ(‚(:‘ Ø¢ùÓ—’ã) proc.memory.percent‚14.36917781829834 cpu‚27.24965997579484 network.recv‚ 8.676794e+06 disk.dm-0.in‚69926.25390625 disk.sda3.in‚42523.43359375( disk.nvme0c0n1.out‚61623.26611328125 disk.sda.in‚42523.43359375 disk.sda1.out‚0& disk.nvme0n1.out‚61623.26611328125$ memory_percent‚68.61639022827148 network.sent‚ 2.2569819e+07 disk.dm-1.in‚42523.43359375 disk.sda3.out‚ 85.3984375 disk.sda.out‚ 85.3984375 proc.cpu.threads‚17) disk./.usagePercent‚32.72816471803289 disk.dm-1.out‚ 85.3984375 disk.drbd2165.in‚0 disk.drbd2165.out‚0# disk.dm-0.out‚61623.26611328125 disk.sda2.out‚0" disk.nvme0n1.in‚69926.25390625# proc.memory.rssMB‚ 2354.24609375$ disk./.usageGB‚487.6733055114746$ disk.nvme0c0n1.in‚69926.25390625 disk.sda1.in‚0 disk.sda2.in‚0' proc.memory.availableMB‚ 5141.890625a>Žl2È(- _wandbruntime‚4611  _runtime‚4611ÂÈóÉÉ(à % _timestamp‚1.7866632624691272e+09 _runtime‚4611.000904008 _step‚1427 & lr-AdamW/pg1‚6.669511095958139e-05 & lr-AdamW/pg2‚6.669511095958139e-05 trainer/global_step‚17349M“&ŒÎÊ(È % _timestamp‚1.7866632624691272e+09 _runtime‚4611.000904008 _step‚1427 & lr-AdamW/pg1‚6.669511095958139e-05 & lr-AdamW/pg2‚6.669511095958139e-05 trainer/global_step‚17349“ I&2Ë(- _wandbruntime‚4611  _runtime‚4611ç ý Å'Ì(¿O ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 4186931067979 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚477709106795520 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 4437243824128 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 4628521206 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚470712286347264 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 5368709120 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 5368709120 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 125829120 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚262 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚54562142 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚429 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚477709106795520 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚58246745 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚58242458 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚474899217415243 % _timestamp‚1.7866632624698083e+09 JDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 5494538240 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚473513458401280 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚477709165845504 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚473513441296384 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚322 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚322 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚58242458 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 122905600 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 122306526 G>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4876 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 4654602240 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 4195665499136 UHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚66779136 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚417191652331520 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 5368709120 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 M@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚59049984 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17104896 D;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚5264 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚412754454642688 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚ 122495285 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 4552982528 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 845178880 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 4517085184 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 4437197688832 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚474899275802623 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚322 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚417191719110656 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 5494538240 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚61478912 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 125829120 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 _step‚1428 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚64252829 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚40003989 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚64252827 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚262 trainer/global_step‚17349 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚223 IBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚67 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚40003991 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚209 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 PCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚58387380 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 KBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚4876 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚46135296 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚262 F?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚60 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚ 122499574 PIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚65 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 4186972415999 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚20643840 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 5494538240 KDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚60 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 4654602240 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚4287 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 IBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚60 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 4195665499136 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚4287 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4289 JBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚665 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚14558151 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 4552982528 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 4195707444224 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚477709165845504 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚64252829 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚473513441296384 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚41945088 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚41945088 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚4289 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚58246745 JDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚477709106795520 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 4437243824128 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 4628521206 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚470712286347264 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 5368709120 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 5368709120 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 125829120 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚262 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚54562142 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚429 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚477709106795520 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚58246745 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚58242458 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚474899217415243 % _timestamp‚1.7866632624698083e+09 JDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 5494538240 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚473513458401280 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚477709165845504 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚473513441296384 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚322 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚322 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚58242458 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 122905600 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 122306526 G>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4876 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 4654602240 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 4195665499136 UHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚66779136 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚417191652331520 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 5368709120 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 M@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚59049984 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17104896 D;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚5264 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚412754454642688 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚ 122495285 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 4552982528 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 845178880 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 4517085184 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 4437197688832 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚474899275802623 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚322 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚417191719110656 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 5494538240 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚61478912 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 125829120 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 _step‚1428 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚64252829 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚40003989 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚64252827 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚262 trainer/global_step‚17349 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚223 IBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚67 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚40003991 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚209 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 PCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚58387380 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 KBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚4876 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚46135296 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.ÏiÀ= current‚262 F?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚60 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚ 122499574 PIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚65 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 4186972415999 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚20643840 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 5494538240 KDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚60 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 4654602240 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚4287 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 IBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚60 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 4195665499136 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚4287 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4289 JBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚665 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚14558151 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 4552982528 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 4195707444224 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚477709165845504 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚64252829 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚473513441296384 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚41945088 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚41945088 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚4289 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚58246745 JDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 _runtime‚4611.166818948 G@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚70 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚4295 E5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5264 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 5368709120 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚473539763634176 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚58245745 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚4293 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚54565100 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚223 B8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 IBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚60 SFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚70970368 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 125829120 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚40006196 K>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59053056 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚41948160 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚14558974 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 5494538240 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚322 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚58245745 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 4552982528 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚41948160 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 DDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654602240 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚417215012363264 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521206 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚477735660201984 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17104896 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚429 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚61478912 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚64256380 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚473539763634176 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 4195938515968 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚14558906 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚322 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 122306526 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚412777522659328 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 4517085184 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17104896 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚473539780739072 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚58250038 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚64256380 I@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚4876 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚262 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 122905600 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 4437489703936 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 4187161640591 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 125829120 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚ 122502125 NGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚68 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚50326528 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚64256382 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚58250038 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚474925633781391 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚470738489180160 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚262 D=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚60 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 4187202988659 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚322 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚ 122506420 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚4293 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 5494538240 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 5368709120 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 5368709120 G@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚60 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚54565170 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 4195896567808 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 122506420 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚412777502015488 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 4654602240 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2¨Ô&Ð(ÎM >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚322 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚58245745 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 4552982528 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚41948160 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 DDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654602240 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚417215012363264 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521206 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚477735660201984 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17104896 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚429 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚61478912 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚64256380 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚473539763634176 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 4195938515968 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚14558906 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚322 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 122306526 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚412777522659328 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 4517085184 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17104896 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚473539780739072 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚58250038 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚64256380 I@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚4876 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚262 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 122905600 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 4437489703936 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 4187161640591 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 125829120 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚ 122502125 NGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚68 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚50326528 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚64256382 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚58250038 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚474925633781391 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚470738489180160 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚262 D=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚60 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 4187202988659 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚322 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚ 122506420 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚4293 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 5494538240 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 5368709120 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 5368709120 G@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚60 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚54565170 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 4195896567808 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 122506420 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚412777502015488 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 4654602240 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚470738472140800 trainer/global_step‚17349 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 122905600 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 831258624 NADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚58387428 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 4195896567808 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 _step‚1429 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 845178880 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 125829120 B9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚5264 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚474925692168819 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚41348068 EDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 _runtime‚4611.166818948 G@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚70 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚4295 E5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5264 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 5368709120 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚473539763634176 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚58245745 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚4293 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚54565100 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚223 B8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 IBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚60 SFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚70970368 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 125829120 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚40006196 K>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59053056 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚41948160 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚14558974 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 5494538240•  ŽÚ2Ñ(- _wandbruntime‚4611  _runtime‚4611•áè=tÒ(î 0loss/FRt1/assignment_loss‚0.4623727100075011 1loss/FRt2/assignment_loss‚0.36434844677304945 .loss/FRt1/detection_loss‚0.501393784534867 $ _timestamp‚1.786663262637455e+09 /loss/FRt2/detection_loss‚0.4361620889317474 &loss/total_loss‚1.7642770302471649 epoch‚43 trainer/global_step‚17349 _runtime‚4611.167644089 _step‚1430¿ÛöÆyÓ(ó .loss/FRt1/detection_loss‚0.501393784534867 $ _timestamp‚1.786663262637455e+09 /loss/FRt2/detection_loss‚0.4361620889317474 &loss/total_loss‚1.7642770302471649 epoch‚43 trainer/global_step‚17349 _runtime‚4611.167644089 _step‚1430 0loss/FRt1/assignment_loss‚0.4623727100075011 1loss/FRt2/assignment_loss‚0.36434844677304945– «cõ<¡Ô(‚(:– ç¢ùÓšÙ á gpu.0.gpu‚60.0 gpu.0.memory‚11, gpu.0.memoryAllocated‚7.0313262939453125) gpu.0.memoryAllocatedBytes‚ 6039863296 gpu.0.temp‚37.0 gpu.0.powerWatts‚204.746' gpu.0.enforcedPowerLimitWatts‚500.0) gpu.0.powerPercent‚40.949200000000005 gpu.0.smClock‚1410 gpu.0.memoryClock‚1593$ gpu.0.correctedMemoryErrors‚1955# gpu.0.uncorrectedMemoryErrors‚0^Ÿòâ™Õ(‚(:Ž ç¢ùÓ¢´Ðã$ memory_percent‚68.61696243286133' proc.memory.availableMB‚ 5141.796875# proc.memory.rssMB‚ 2354.24609375) proc.memory.percent‚14.36917781829834 cpu‚27.249333053013352$ disk./.usageGB‚487.6733627319336$ disk.nvme0c0n1.in‚69942.30859375( disk.nvme0c0n1.out‚61629.09423828125" disk.nvme0n1.in‚69942.30859375 proc.cpu.threads‚17( disk./.usagePercent‚32.7281685581459 disk.dm-1.in‚42523.43359375 disk.drbd2165.in‚0 disk.drbd2165.out‚0 disk.dm-0.in‚69942.30859375 disk.sda2.in‚0& disk.nvme0n1.out‚61629.09423828125 network.sent‚ 2.2641486e+07 network.recv‚ 8.705534e+06# disk.dm-0.out‚61629.09423828125 disk.sda3.in‚42523.43359375 disk.sda3.out‚ 85.609375 disk.sda.out‚ 85.609375 disk.sda1.in‚0 disk.sda1.out‚0 disk.dm-1.out‚ 85.609375 disk.sda.in‚42523.43359375 disk.sda2.out‚0j§´2Ö(- _wandbruntime‚4623  _runtime‚4623¯¨É×(à _step‚1431 & lr-AdamW/pg1‚6.423920217430435e-05 & lr-AdamW/pg2‚6.423920217430435e-05 trainer/global_step‚17399 % _timestamp‚1.7866632750784452e+09 _runtime‚4623.608500625ô@‹ÎØ(È & lr-AdamW/pg1‚6.423920217430435e-05 & lr-AdamW/pg2‚6.423920217430435e-05 trainer/global_step‚17399 % _timestamp‚1.7866632750784452e+09 _runtime‚4623.608500625 _step‚1431— š2Ù(- _wandbruntime‚4623  _runtime‚4623 €”8Ú(¾O GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 4552982528 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚413906843926528 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 122306526 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 trainer/global_step‚17399 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚474829575290880 PCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚59049984 JDeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4289 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚64430477 IBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚67 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚262 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17104896 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚ 122842326 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚14598830 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚209 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 125829120 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚479036869500928 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚474829558185984 PIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚65 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚20643840 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 G>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4876 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚474829558185984 D;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚5264 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 5494538240 IBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚60 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚479036810450944 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚41945088 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚54713136 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 BDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚476220068902651 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 4628521206 % _timestamp‚1.7866632750794644e+09 @7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚5264 F>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚479036869500928 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚41945088HMuéÉ'Û(ÃO D;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚5264 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 5494538240 IBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚60 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚479036810450944 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚41945088 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚54713136 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 BDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚476220068902651 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 4628521206 % _timestamp‚1.7866632750794644e+09 @7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚5264 F>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚479036869500928 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚41945088 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 4552982528 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚413906843926528 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 122306526 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 trainer/global_step‚17399 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚474829575290880 PCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚59049984 JDeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4289 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚64430477 IBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚67 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚262 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17104896 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚ 122842326 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚14598830 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚209 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 125829120 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚479036869500928 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚474829558185984 PIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚65 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚20643840 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 G>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4876 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚474829558185984˜ ’1Ð2Ü(- _wandbruntime‚4623  _runtime‚4623kW½µÏ&Ý(ÉM REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17104896 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 4207483333632 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚ 122844877 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚64434030 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚429 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚418379681961472 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 125829120 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚262 NADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚59053056 H@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚665 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚58415140 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚418379613088256 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚48229376 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚61478912 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚4295 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 4654602240 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚476246485268799 D=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚60 % _timestamp‚1.7866632752432685e+09 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 122905600 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚223 BDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521206 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 4517085184 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 4552982528 IBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚60 trainer/global_step‚17399 _runtime‚4623.773550897 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚479063422910464 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚413929891299328 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚322 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚58410847 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654602240 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 122849172 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚40116446 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 5368709120 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚41348068 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 845178880 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚322 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 122905600 B9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚5264 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 5494538240 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚479063363857408 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 122306526 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 5368709120 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 5368709120 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚ 122844877 K>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59053056 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚40116444 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 5494538240 DDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚64434030 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚474855880523776 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 125829120 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚64434032 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 4552982528 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 4207525281792 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17104896 E5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5264 PDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚54716079 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚472047761817600 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 G@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚60 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 4207483333632 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚54716160 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 4449742432768 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚476246543656227w¯º‡%Þ(ÎM XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 5368709120 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 5368709120 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚ 122844877 K>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59053056 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚40116444 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 5494538240 DDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚64434030 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚474855880523776 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 125829120 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚64434032 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 4552982528 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 4207525281792 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17104896 E5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5264 PDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚54716079 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚472047761817600 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 G@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚60 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 4207483333632 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚54716160 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 4449742432768 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚476246543656227 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17104896 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 4207483333632 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚ 122844877 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚64434030 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚429 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚418379681961472 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 125829120 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚262 NADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚59053056 H@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚665 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚58415140 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚418379613088256 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚48229376 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚61478912 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚4295 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 4654602240 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚476246485268799 D=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚60 % _timestamp‚1.7866632752432685e+09 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 122905600 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚223 BDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521206 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 4517085184 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 4552982528 IBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚60 trainer/global_step‚17399 _runtime‚4623.773550897 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚479063422910464 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚413929891299328 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚322 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚58410847 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654602240 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 122849172 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚40116446 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 5368709120 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚41348068 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 845178880 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚322 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 122905600 B9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚5264 4.DeviceStatsMonitor.on_train_batch_end/num_oê,ŸÓMoms‚0 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 5494538240 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚479063363857408 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 122306526 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0™ g娚2ß(- _wandbruntime‚4623  _runtime‚4623¶¦µ„wà(ñ 0loss/FRt1/detection_loss‚0.49374498612424333 /loss/FRt2/detection_loss‚0.4856026744115544 % _timestamp‚1.7866632752442708e+09 _step‚1434 0loss/FRt1/assignment_loss‚0.4931350180115368 1loss/FRt2/assignment_loss‚0.40205767032099293 &loss/total_loss‚1.8745403488683274 epoch‚43 trainer/global_step‚17399 _runtime‚4623.774090017^’|á(ö _runtime‚4623.774090017 0loss/FRt1/detection_loss‚0.49374498612424333 /loss/FRt2/detection_loss‚0.4856026744115544 % _timestamp‚1.7866632752442708e+09 _step‚1434 0loss/FRt1/assignment_loss‚0.4931350180115368 1loss/FRt2/assignment_loss‚0.40205767032099293 &loss/total_loss‚1.8745403488683274 epoch‚43 trainer/global_step‚17399š gƒÏe–â(‚(:‹ ö¢ùÓ¾³‰á gpu.0.gpu‚64.0 gpu.0.memory‚12, gpu.0.memoryAllocated‚7.0313262939453125) gpu.0.memoryAllocatedBytes‚ 6039863296 gpu.0.temp‚36.0 gpu.0.powerWatts‚174.881' gpu.0.enforcedPowerLimitWatts‚500.0 gpu.0.powerPercent‚34.9762 gpu.0.smClock‚1410 gpu.0.memoryClock‚1593$ gpu.0.correctedMemoryErrors‚1955# gpu.0.uncorrectedMemoryErrors‚0¬09šã(‚(: ö¢ùÓ ¾Øâ$ disk.nvme0c0n1.in‚69966.10546875 disk.sda.in‚42523.43359375 disk.sda.out‚ 85.8046875# proc.memory.rssMB‚ 2354.24609375) proc.memory.percent‚14.36917781829834 network.sent‚ 2.2716701e+07% disk.nvme0n1.out‚61673.7119140625 disk.dm-1.out‚ 85.8046875 disk.sda3.out‚ 85.8046875' disk.nvme0c0n1.out‚61673.7119140625 disk.sda1.in‚0$ memory_percent‚68.61696243286133' proc.memory.availableMB‚ 5141.796875% disk./.usageGB‚487.67320251464844 disk.sda2.out‚0" disk.nvme0n1.in‚69966.10546875 disk.dm-1.in‚42523.43359375 disk.sda1.out‚0 cpu‚27.248918481013778 disk.sda2.in‚0 disk.drbd2165.in‚0 disk.dm-0.in‚69966.10546875" disk.dm-0.out‚61673.7119140625 disk.sda3.in‚42523.43359375 proc.cpu.threads‚17 network.recv‚ 8.73812e+06) disk./.usagePercent‚32.72815780582947 disk.drbd2165.out‚0„ FÞ2ä(- _wandbruntime‚4636  _runtime‚4636Ü¡ÐTÇå(Á % _timestamp‚1.7866632877428126e+09 _runtime‚4636.273002985 _step‚1435 % lr-AdamW/pg1‚6.18273385236231e-05 % lr-AdamW/pg2‚6.18273385236231e-05 trainer/global_step‚17449U[^^Ìæ(Æ trainer/global_step‚17449 % _timestamp‚1.7866632877428126e+09 _runtime‚4636.273002985 _step‚1435 % lr-AdamW/pg1‚6.18273385236231e-05 % lr-AdamW/pg2‚6.18273385236231e-05› ð|ß”2ç(- _wandbruntime‚4636  _runtime‚4636$á~Å'è(¿O M@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚59049984 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 4552982528 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 4517085184 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚480364573156352 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚476145675075584 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 122905600 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚4287 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 122306526 UHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚62584832 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚473330865700864 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 5494538240 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 5494538240 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 5368709120 trainer/global_step‚17449 JDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚477540978777439 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 4210096037215 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 125829120 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚415059191922688 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚4289 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 D;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚5264 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚20643840 BDeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4289 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 5494538240 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚262 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 @7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚5264 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚322 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚262 % _timestamp‚1.7866632877438946e+09 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 4628521206 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 4654602240 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚476145692180480 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17104896 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚58572662 PCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚58387380 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 F>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 KBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚4876 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚14640007 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚322 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚ 123180789 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 4218839030784 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 845178880 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 4461829533184 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚58576949 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 125829120 JBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚665 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 4654602240 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚64608129 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 KDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚60 F?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚60 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚322 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚480364514106368 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚480364573156352 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚480364514106368 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 4218880975872 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚419520979514880 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 4461787592192 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚64608129 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚4287 PCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚59049984 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 4218880975872 IBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚69 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚14640074 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚61478912 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚477540920390059 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚58576949 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚40224489 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 831258624 G>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4876 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚41940992 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 4210054689195 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 125829120 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 4218839030784 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚429 PIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚67 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚41945088 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 4552982528 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚476145692180480 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 _runtime‚4636.277878144 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚58572662 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚41945088 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 122905600 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚209 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚419521042099712 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚262 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚64608127 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚41348020ììßÊ'é(ÄO UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 5368709120 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚54864565 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚40224491 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚223 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚415059212566528 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 IBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚60 _step‚1436 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚64608127 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚476145675075584 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4289 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 5494538240 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚262 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 @7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚5264 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚322 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚262 % _timestamp‚1.7866632877438946e+09 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 4628521206 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 4654602240 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚476145692180480 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17104896 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚58572662 PCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚58387380 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 F>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 KBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚4876 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚14640007 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚322 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚ 123180789 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 4218839030784 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 845178880 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 4461829533184 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚58576949 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 125829120 JBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚665 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 4654602240 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚64608129 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 KDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚60 F?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚60 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚322 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚480364514106368 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚480364573156352 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚480364514106368 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 4218880975872 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚419520979514880 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 4461787592192 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚64608129 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚4287 PCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚59049984 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 4218880975872 IBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚69 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚14640074 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚61478912 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚477540920390059 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚58576949 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚40224489 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 831258624 G>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4876 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚41940992 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 4210054689195 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 125829120 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 4218839030784 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚429 PIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚67 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚41945088 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 4552982528 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚476145692180480 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 _runtime‚4636.277878144 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚58572662 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚41945088 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 122905600 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚209 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚419521042099712 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚262 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚64608127 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚41348020 M@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚59049984 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 4552982528 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 4517085184 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚480364573156352 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚476145675075584 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 122905600 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚4287 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 122306526 UHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚62584832 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚473330865700864 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 5494538240 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 5494538240 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 5368709120 trainer/global_step‚17449 JDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚477540978777439 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 4210096037215 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 125829120 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚415059191922688 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚4289 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 D;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚5264 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚20643840 B6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚322 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 4219112047616 EDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚54867531 K>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59053056 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17104896 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚64611680 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚20643840 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚262 B9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚5264 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 5368709120 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚4293 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 123191924 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚4293 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚61478912 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 NADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚58387428 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚64611680 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚429 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚322 G@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚65 SFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚62581760 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚477567395143635 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521206 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚4295 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚ 123187629 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 4219112047616 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚41948160 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚473357068533760 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚41348068 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚64611682 EDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 _runtime‚4636.436342162 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654602240 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚476171997413376 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚209 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚419544273131008 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚477567336756207 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚58575949 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚40226696 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 4552982528 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 122905600 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚58580242 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 4552982528 D6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚322 G@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚60 % _timestamp‚1.7866632879059105e+09 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚476172014518272 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 trainer/global_step‚17449 H@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚665 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 4654602240 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚ 123187629 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚64611682 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚322 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚58580242 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚415082239295488 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 4210326609875 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 5494538240 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚223 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚415082259939328 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 4517085184 SEDeviceStatsMonitor.on_trainà—1‹_batch_end/requested_bytes.small_pool.peak‚ 122306526 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 >5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5264 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚419544335712768 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚480391067512832 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 4219070099456 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 D=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚60b·@ÉÔ&ì(ÎM B9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚5264 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 5368709120 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚4293 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 123191924 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚4293 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚61478912 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 NADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚58387428 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚64611680 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚429 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚322 G@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚65 SFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚62581760 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚477567395143635 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521206 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚4295 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚ 123187629 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 4219112047616 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚41948160 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚473357068533760 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚41348068 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚64611682 EDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 _runtime‚4636.436342162 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654602240 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚476171997413376 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚209 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚419544273131008 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚477567336756207 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚58575949 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚40226696 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 4552982528 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 122905600 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚58580242 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 4552982528 D6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚322 G@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚60 % _timestamp‚1.7866632879059105e+09 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚476172014518272 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 trainer/global_step‚17449 H@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚665 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 4654602240 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚ 123187629 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚64611682 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚322 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚58580242 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚415082239295488 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 4210326609875 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 5494538240 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚223 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚415082259939328 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 4517085184 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 122306526 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 >5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5264 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚419544335712768 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚480391067512832 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 4219070099456 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 D=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚60 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚ 123191924 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚480391126565888 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚473357051494400 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚322 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 4219112047616 EDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚54867531 K>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59053056 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17104896 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚64611680 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚20643840 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚262 ‰Iùð2í(- _wandbruntime‚4636  _runtime‚4636¢Š&tuî(ï /loss/FRt2/detection_loss‚0.4890787425885234 % _timestamp‚1.7866632879069493e+09 _step‚1438 0loss/FRt1/assignment_loss‚0.5093558951213767 /loss/FRt1/detection_loss‚0.5112838520078731 &loss/total_loss‚1.8924124538894027 epoch‚43 trainer/global_step‚17449 _runtime‚4636.437346311 0loss/FRt2/assignment_loss‚0.3826939641716296;a|zï(ô trainer/global_step‚17449 _runtime‚4636.437346311 0loss/FRt2/assignment_loss‚0.3826939641716296 /loss/FRt2/detection_loss‚0.4890787425885234 % _timestamp‚1.7866632879069493e+09 _step‚1438 0loss/FRt1/assignment_loss‚0.5093558951213767 /loss/FRt1/detection_loss‚0.5112838520078731 &loss/total_loss‚1.8924124538894027 epoch‚43ž YŒÜ2ð(- _wandbruntime‚4649  _runtime‚4649_¬|¥Èñ( & lr-AdamW/pg1‚5.945967491703314e-05 & lr-AdamW/pg2‚5.945967491703314e-05 trainer/global_step‚17499 $ _timestamp‚1.786663300478994e+09 _runtime‚4649.009031436 _step‚1439ÀC‚ÃÍò(Ç _runtime‚4649.009031436 _step‚1439 & lr-AdamW/pg1‚5.945967491703314e-05 & lr-AdamW/pg2‚5.945967491703314e-05 trainer/global_step‚17499 $ _timestamp‚1.786663300478994e+09Ÿ -k–2ó(- _wandbruntime‚4649  _runtime‚4649¦mË!Ä'ô(¾O F>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚477461791965184 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 4552982528 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚416211581206528 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 4230425796608 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚4287 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 122306526 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 125829120 _step‚1440 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 JDeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4289 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚58737764 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 4654602240 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 4654602240 UHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚64681984 F?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚60 IBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚60 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚58742051 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚322 BDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚481692217761792 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 4474076813824 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚41348020 KDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚60 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚64785777 IBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚81 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 5368709120 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚58737764 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚55015911 G>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4876 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 122905600 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚14681091 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚209 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚44038144 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 4221616499803 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚ 123523541 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚20643840 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚61478912 _runtime‚4649.013895625 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 4628521206 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 4517085184 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 5368709120 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚477461809070080 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17104896 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17104896 PIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚79 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚474640155377664 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 5494538240 PCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚59049984 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 831258624 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚474640172417024 C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚4289 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 4552982528 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚64785779 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚416211560562688 PCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚58387380 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 trainer/global_step‚17499 M@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚59049984 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 JBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚665 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 $ _timestamp‚1.786663300479966e+09 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚481692217761792 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚477461809070080 JDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚477461791965184 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 4552982528 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚416211581206528 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 4230425796608 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚4287 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 122306526 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 125829120 _step‚1440 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 JDeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4289 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚58737764 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 4654602240 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 4654602240 UHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚64681984 F?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚60 IBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚60 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚58742051 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚322 BDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚481692217761792 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 4474076813824 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚41348020 KDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚60 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚64785777 IBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚81 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 5368709120 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚58737764 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚55015911 G>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4876 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 122905600 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚14681091 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚209 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚44038144 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 4221616499803 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚ 123523541 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚20643840 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚61478912 _runtime‚4649.013895625 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 4628521206 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 4517085184 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 5368709120 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚477461809070080  ±ò¾I2ö(- _wandbruntime‚4649  _runtime‚4649$Ú£“m÷(ÈM NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚64789332 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 4230698813440 SFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚62581760 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚262 TEDeviceStatsMonitor.o\¯'a%n_train_batch_end/allocated_bytes.large_pool.peak‚ 4552982528 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚209 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 4474366043648 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚58741051 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚58741051 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚40336946 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚322 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚474666341171200 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚41348068 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚ 123534676 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 122905600 EDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚420708994622976 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚322 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚262 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚64789332 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 trainer/global_step‚17499 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚477488114302976 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚64789330 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚4293 NGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚68 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 5494538240 P8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 122905600 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚223 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚474666358210560 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚4295 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚55018900 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 5368709120 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚262 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 4654602240 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚416234628579328 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚481718830221312 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚41937920 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 4474324105728 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚478888188243615 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚477488114302976 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 125829120 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚55018970 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚429 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 125829120 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 4230656865280 NADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚59053056 B9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚5264 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚14681956 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 4221847072415 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 K>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59053056 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 4552982528 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 4230698813440 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚322 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17104896 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 845178880 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚416234607935488 G@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚70 NADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚58387428 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 5494538240 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚481718830221312 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚477488131407872 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 % _timestamp‚1.7866633006592433e+09 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚4293 I@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚4876 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚14682024 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 EDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 123534676 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚58745344 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚61478912 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 4517085184 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 4221888420483 _step‚1441 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚41948160 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654602240 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚ 123530381 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 831258624 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521206 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚64789330 D5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5264 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 4230656865280 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 122306526 _runtime‚ 4649.18961059 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17104896 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚58745344 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚481718771168256 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 125829120±^ÅÄÓ&ø(ÍM A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚4295 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚55018900 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 5368709120 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚262 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 4654602240 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚416234628579328 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚481718830221312 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚41937920 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 4474324105728 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚478888188243615 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚477488114302976 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 125829120 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚55018970 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚429 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 125829120 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 4230656865280 NADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚59053056 B9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚5264 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚14681956 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 4221847072415 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 K>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59053056 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 4552982528 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 4230698813440 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚322 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17104896 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 845178880 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚416234607935488 G@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚70 NADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚58387428 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 5494538240 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚481718830221312 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚477488131407872 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 % _timestamp‚1.7866633006592433e+09 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚4293 I@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚4876 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚14682024 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 EDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 123534676 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚58745344 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚61478912 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 4517085184 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 4221888420483 _step‚1441 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚41948160 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654602240 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚ 123530381 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 831258624 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521206 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚64789330 D5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5264 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 4230656865280 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 122306526 _runtime‚ 4649.18961059 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17104896 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚58745344 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚481718771168256 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 125829120 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚64789332 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 4230698813440 SFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚62581760 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚262 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 4552982528 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚209 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 4474366043648 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚58741051 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚58741051 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚40336946 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚322 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚474666341171200 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚41348068 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚ 123534676 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 122905600 EDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚420708994622976 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚322 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚262 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚64789332 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 trainer/global_step‚17499 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚477488114302976 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚64789330 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚4293 NGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚68 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 5494538240 P8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 122905600 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚223 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚474666358210560¡ T^3ò2ù(- _wandbruntime‚4649  _runtime‚4649"Êáíwú(ñ _runtime‚4649.190739816 _step‚1442 1loss/FRt2/assignment_loss‚0.38266527029932834 &loss/total_loss‚1.8404902849737252 epoch‚43 trainer/global_step‚17499 % _timestamp‚1.7866633006608238e+09 0loss/FRt1/assignment_loss‚0.5041082196116681 /loss/FRt1/detection_loss‚0.5038739062704892 0loss/FRt2/detection_loss‚0.44984288879223966ïÍÅÄ|û(ö trainer/global_step‚17499 % _timestamp‚1.7866633006608238e+09 0loss/FRt1/assignment_loss‚0.5041082196116681 /loss/FRt1/detection_loss‚0.5038739062704892 0loss/FRt2/detection_loss‚0.44984288879223966 _runtime‚4649.190739816 _step‚1442 1loss/FRt2/assignment_loss‚0.38266527029932834 &loss/total_loss‚1.8404902849737252 epoch‚43¢ ‚Ëùf•ü(‚(:Š …£ùÓÁɳá gpu.0.gpu‚52.0 gpu.0.memory‚9, gpu.0.memoryAllocated‚7.0313262939453125) gpu.0.memoryAllocatedBytes‚ 6039863296 gpu.0.temp‚34.0 gpu.0.powerWatts‚124.143' gpu.0.enforcedPowerLimitWatts‚500.0 gpu.0.powerPercent‚24.8286 gpu.0.smClock‚1410 gpu.0.memoryClock‚1593$ gpu.0.correctedMemoryErrors‚1955# gpu.0.uncorrectedMemoryErrors‚0ÞAò›ý(‚(: …£ùÓ§¥ýã disk.drbd2165.in‚0 disk.dm-0.in‚ 70006.9765625$ memory_percent‚68.61646175384521) disk./.usagePercent‚32.72815754982194! disk.nvme0n1.in‚ 70006.9765625 disk.drbd2165.out‚0) proc.memory.availableMB‚ 5141.87890625) proc.memory.percent‚14.36917781829834 disk.sda.in‚42523.43359375 disk.sda1.in‚0 disk.sda1.out‚0 disk.sda2.out‚0% disk.nvme0n1.out‚61706.9423828125 disk.dm-1.out‚ 86.10546875" disk.dm-0.out‚61706.9423828125# proc.memory.rssMB‚ 2354.24609375 network.sent‚ 2.2791814e+07 network.recv‚ 8.771031e+06$ disk./.usageGB‚487.6731986999512 disk.sda3.out‚ 86.10546875 disk.sda.out‚ 86.10546875 cpu‚27.24858637485568 proc.cpu.threads‚17 disk.sda3.in‚42523.43359375# disk.nvme0c0n1.in‚ 70006.9765625' disk.nvme0c0n1.out‚61706.9423828125 disk.sda2.in‚0 disk.dm-1.in‚42523.43359375Т«­2þ(- _wandbruntime‚4661  _runtime‚4661ZmÉÿ(à _runtime‚4661.270119092 _step‚1443 & lr-AdamW/pg1‚5.713636342514339e-05 & lr-AdamW/pg2‚5.713636342514339e-05 trainer/global_step‚17549 % _timestamp‚1.7866633127401137e+09/R¬Ô΀)È & lr-AdamW/pg1‚5.713636342514339e-05 & lr-AdamW/pg2‚5.713636342514339e-05 trainer/global_step‚17549 % _timestamp‚1.7866633127401137e+09 _runtime‚4661.270119092 _step‚1443£ ûJV2)- _wandbruntime‚4661  _runtime‚4661Ø Ú{Å'‚)¿O TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚14722452 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 5494538240 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 4242054507520 KBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚4876 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 4233219658431 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚58902866 JDeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4289 D;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚5264 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚64963429 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 4233178310411 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚322 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 4242012562432 IBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚79 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 845178880 BDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4876 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 125829120 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚322 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 125829120 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚478777925959680 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚480182681752255 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 4628521206 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚40444989 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚209 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚262 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚483019921417216 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚429 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 F>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚478777908854784 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 @7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚5264 KDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚60 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚4287 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚41348020Fšò^\ƒ)ÄO QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚55167520 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 5368709120 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚483019980467200 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚‘Œó n& 4242012562432 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚223 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚322 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 125829120 C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚4289 G>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4876 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 125829120 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚322 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 125829120 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚478777925959680 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚480182681752255 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 4628521206 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚40444989 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚209 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚262 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚483019921417216 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚429 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 F>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚478777908854784 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 @7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚5264 KDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚60 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚4287 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚41348020 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚14722452 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 5494538240 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 4242054507520 KBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚4876 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 4233219658431 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚58902866 JDeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4289 D;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚5264 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚64963429 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 4233178310411 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚322 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 4242012562432 IBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚79 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 845178880 B6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚322 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚64966980 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚322 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚262 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚ 123873133 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚64966982 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 123877428 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚480209039731023 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 4233450231091 K>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59053056 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚478804248297472 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 122905600 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚58910446 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚20643840 EDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚55170451 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 5494538240 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚417386976575488 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 4486657902592 D=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚60 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 122905600 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚421873588345856 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 >5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5264 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 4486611770368 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚475975630848000 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 125829120 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚262 IBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚60 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚61478912 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚322 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521206 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚209 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 4242243631104 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654602240 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚41948160 B9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚5264 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚40447196 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚483046533876736 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚475975647887360 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 4517085184 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 PDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17104896 EDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17104896 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚14723326 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚262 $ _timestamp‚1.786663312903104e+09 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 5494538240pñŽÓ&†)ÍM EDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚55170451 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 5494538240 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚417386976575488 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 4486657902592 D=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚60 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 122905600 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚421873588345856 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 >5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5264 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 4486611770368 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚475975630848000 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 125829120 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚262 IBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚60 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚61478912 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚322 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521206 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚209 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 4242243631104 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654602240 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚41948160 B9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚5264 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚40447196 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚483046533876736 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚475975647887360 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 4517085184 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 PDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17104896 EDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17104896 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚14723326 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚262 $ _timestamp‚1.786663312903104e+09 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 5494538240 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚58906153 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 NADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚58387428 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚ 123877428 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚41948160 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚58906153 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚40447194 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚322 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚64966980 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚322 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚262 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚ 123873133 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚64966982 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 123877428 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚480209039731023 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 4233450231091 K>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59053056 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚478804248297472 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 122905600 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚58910446 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚20643840¥ ðxÃ2‡)- _wandbruntime‚4661  _runtime‚46619ÿe(xˆ)ò 1loss/FRt1/assignment_loss‚0.49513967353534283 1loss/FRt2/assignment_loss‚0.38725578301264724 &loss/total_loss‚1.8764755417879415 % _timestamp‚1.7866633129041402e+09 _runtime‚4661.433974161 /loss/FRt1/detection_loss‚0.5124020168290412 0loss/FRt2/detection_loss‚0.48167806841091015 epoch‚43 trainer/global_step‚17549 _step‚14468O‡}‰)÷ &loss/total_loss‚1.8764755417879415 % _timestamp‚1.7866633129041402e+09 _runtime‚4661.433974161 /loss/FRt1/detection_loss‚0.5124020168290412 0loss/FRt2/detection_loss‚0.48167806841091015 epoch‚43 trainer/global_step‚17549 _step‚1446 1loss/FRt1/assignment_loss‚0.49513967353534283 1loss/FRt2/assignment_loss‚0.38725578301264724¦ ‚7‚”Š)‚(:‰ ”£ùÓƨ á gpu.0.gpu‚0.0 gpu.0.memory‚0, gpu.0.memoryAllocated‚7.0313262939453125) gpu.0.memoryAllocatedBytes‚ 6039863296 gpu.0.temp‚36.0 gpu.0.powerWatts‚129.029' gpu.0.enforcedPowerLimitWatts‚500.0 gpu.0.powerPercent‚25.8058 gpu.0.smClock‚1410 gpu.0.memoryClock‚1593$ gpu.0.correctedMemoryErrors‚1955# gpu.0.uncorrectedMemoryErrors‚0¿óy¨œ‹)‚(:‘ ”£ùÓ¾¢ä disk.dm-0.in‚70016.42578125" disk.dm-0.out‚61753.1611328125 disk.sda3.out‚ 86.3828125$ disk./.usageGB‚487.1470146179199% disk.nvme0n1.out‚61753.1611328125 disk.sda3.in‚42570.16015625$ disk.nvme0c0n1.in‚70016.42578125' disk.nvme0c0n1.out‚61753.1611328125 disk.sda.out‚ 86.3828125$ memory_percent‚68.61751079559326) proc.memory.availableMB‚ 5141.70703125 network.recv‚ 8.799574e+06) disk./.usagePercent‚32.69284489457897 disk.sda1.out‚0# proc.memory.rssMB‚ 2354.24609375 cpu‚27.248215264142008 network.sent‚ 2.2883668e+07 disk.drbd2165.in‚0 disk.drbd2165.out‚0 disk.sda.in‚42570.16015625 disk.sda1.in‚0) proc.memory.percent‚14.36917781829834 proc.cpu.threads‚17 disk.sda2.in‚0 disk.sda2.out‚0" disk.nvme0n1.in‚70016.42578125 disk.dm-1.in‚42570.16015625 disk.dm-1.out‚ 86.3828125媊2Œ)- _wandbruntime‚4673  _runtime‚4673>¾Ê)Ä _runtime‚4673.465353164 _step‚1447 ' lr-AdamW/pg1‚5.4857553269909406e-05 ' lr-AdamW/pg2‚5.4857553269909406e-05 trainer/global_step‚17599 $ _timestamp‚1.786663324935344e+09kèÍÈÏŽ)É _step‚1447 ' lr-AdamW/pg1‚5.4857553269909406e-05 ' lr-AdamW/pg2‚5.4857553269909406e-05 trainer/global_step‚17599 $ _timestamp‚1.786663324935344e+09 _runtime‚4673.465353164§ ‘ÝžÀ2)- _wandbruntime‚4673  _runtime‚4673h5Ö÷ )¿O R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚484347625072640 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚4287 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚429 ICDeviceStatsMonitor¼&.on_train_batch_start/oversize_segments.allocated‚0 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 122306526 JDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 PCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚59049984 KBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚4876 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚65141077 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚223 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚322 KDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚60 F?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚60 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 122905600 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 4253599328256 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚40555239 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚481503474852283 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17104896 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚262 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 4552982528 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 4253641273344 JDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 5494538240 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚484347684122624 IBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚81 PCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚58387380 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚41348020 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 4244740121019 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚322 _step‚1448 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚59067968 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 4244781469039 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚41945088 UHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚68876288 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚423014963417088 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 5494538240 _runtime‚4673.469866398 M@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚59049984 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 125829120 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚322 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 5368709120 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚65141079 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚484347625072640 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚41945088 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚4287 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 831258624 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚48232448 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 5494538240 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚ 124209045 G>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4876 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 5368709120 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚480094025744384 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4289 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚40555241 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚14763566 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚262 trainer/global_step‚17599 % _timestamp‚1.7866633249362495e+09 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 4253641273344 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 4654602240 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 845178880™ íBÊ'‘)ÄO TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17104896 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚262 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 4552982528 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 4253641273344 JDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 5494538240 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚484347684122624 IBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚81 PCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚58387380 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚41348020 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 4244740121019 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚322 _step‚1448 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚59067968 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 4244781469039 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚41945088 UHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚68876288 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚423014963417088 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 5494538240 _runtime‚4673.469866398 M@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚59049984 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 125829120 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚322 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 5368709120 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚65141079 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚484347625072640 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚41945088 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚4287 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 831258624 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚48232448 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 5494538240 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚ 124209045 G>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4876 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 5368709120 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚480094025744384 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4289 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚40555241 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚14763566 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚262 trainer/global_step‚17599 % _timestamp‚1.7866633249362495e+09 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 4253641273344 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 4654602240 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 845178880 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚484347625072640 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚4287 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚429 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 122306526 JDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 PCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚59049984 KBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚4876 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚65141077 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚223 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚322 KDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚60 F?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚60 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 122905600 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 4253599328256 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚40555239 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚481503474852283¨ nõm&2’)- _wandbruntime‚4673  _runtime‚4673•øÓ5Ï&“)ÉM NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚59075548 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚4293 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 845178880 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 4253872345088 SFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚58387456 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚423038259843072 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚4293 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 4244970693631 trainer/global_step‚17599 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 124220180 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚59075548 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 831258624 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚322 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 122905600 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17104896 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 4552982528 G@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚65 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚423038318230528 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚481529891218431 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚481529949605859 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 5494538240 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚322 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚4295 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 5368709120 _step‚1449 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚429 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚477284920524800 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚59071255 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚484374178479104 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 122905600 B9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚5264 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521206 K>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59053056 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚55321911 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚40557446 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚477284937564160 _runtime‚4673.633984921 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚41948160 D=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚60 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚ 124215885 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 5368709120 >5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5264 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17104896 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚322 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚65144632 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚484374237532160 H@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚665 NGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚63 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 4498914627584 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 122306526 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 5494538240 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚40557444 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 4253830396928 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 4498952371200 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚484374237532160 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚37743616 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚480120348082176 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 4253830396928 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚480120365187072 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚14764465 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 4517085184 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚322 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚262 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 4253872345088 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚59071255 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 G@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚60 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 4552982528 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚41948160 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚209 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 125829120 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚65144632 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 IBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚60 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚14764402 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚41348068 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 125829120 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚ 124220180 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 4654602240 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚480120365187072 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚55321846 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 NADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚58387428 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 5494538240 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚65144630 DDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654602240 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚418539345215488 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚418539365859328 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚262#’ܳV ”)ÎM >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚322 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 122905600 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17104896 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 4552982528 G@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚65 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚423038318230528 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚481529891218431 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚481529949605859 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 5494538240 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚322 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚4295 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 5368709120 _step‚1449 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚429 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚477284920524800 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚59071255 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚484374178479104 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 122905600 B9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚5264 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521206 K>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59053056 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚55321911 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚40557446 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚477284937564160 _runtime‚4673.633984921 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚41948160 D=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚60 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚ 124215885 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 5368709120 >5DeÓ>Ë~viceStatsMonitor.on_train_batch_end/active.all.peak‚5264 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17104896 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚322 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚65144632 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚484374237532160 H@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚665 NGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚63 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 4498914627584 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 122306526 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 5494538240 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚40557444 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 4253830396928 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 4498952371200 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚484374237532160 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚37743616 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚480120348082176 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 4253830396928 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚480120365187072 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚14764465 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 4517085184 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚322 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚262 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 4253872345088 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚59071255 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 G@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚60 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 4552982528 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚41948160 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚209 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 125829120 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚65144632 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 IBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚60 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚14764402 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚41348068 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 125829120 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚ 124220180 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 4654602240 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚480120365187072 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚55321846 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 NADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚58387428 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 5494538240 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚65144630 DDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654602240 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚418539345215488 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚418539365859328 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚262 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚59075548 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚4293 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 845178880 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 4253872345088 SFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚58387456 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚423038259843072 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚4293 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 4244970693631 trainer/global_step‚17599 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 124220180 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚59075548 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 831258624© •4<2•)- _wandbruntime‚4673  _runtime‚4673ƒŒ\›x–)ò 1loss/FRt2/assignment_loss‚0.36658336878551845 0loss/FRt1/detection_loss‚0.46700078376554693 /loss/FRt2/detection_loss‚0.4980048501526936 &loss/total_loss‚1.7883673485162102 epoch‚43 % _timestamp‚1.7866633251047602e+09 _runtime‚4673.634553737 _step‚1450 1loss/FRt1/assignment_loss‚0.45677834581245114 trainer/global_step‚17599Õ²ñ¼}—)÷ &loss/total_loss‚1.7883673485162102 epoch‚43 % _timestamp‚1.7866633251047602e+09 _runtime‚4673.634553737 _step‚1450 1loss/FRt1/assignment_loss‚0.45677834581245114 trainer/global_step‚17599 1loss/FRt2/assignment_loss‚0.36658336878551845 0loss/FRt1/detection_loss‚0.46700078376554693 /loss/FRt2/detection_loss‚0.4980048501526936ª ç‚ëç2˜)- _wandbruntime‚4678  _runtime‚4678|eTà™)Ú /Purity/*FRt/event_purity‚0.4669463336467743 .Purity/*FRt/FRt_purity‚0.48037365078926086 -Purity/2FRt/FRt_purity‚0.5663777589797974 /particle/accuracy_1_of_2‚0.9226521849632263 2Purity/*FRt/event_proportion‚0.669805645942688 Purity/0FRt/event_purity‚1 +jet/accuracy_2_of_2‚0.21150988340377808 +validation_accuracy‚0.21150988340377808 .Purity/1FRt/FRt_purity‚0.46925240755081177 )jet/accuracy_1_of_2‚0.577630341053009 particle/accuracy_1_of_1‚1 epoch‚43 trainer/global_step‚17599 _step‚1451 _runtime‚4678.527869382 0Purity/1FRt/event_purity‚0.46925240755081177 2Purity/2FRt/event_proportion‚0.040062315762043 +particle/specificity‚0.6440638303756714 'particle/f_score‚0.5539261698722839 *loss/val_total_loss‚2.5239334406896683 *jet/accuracy_1_of_1‚0.3104802668094635 (particle/accuracy‚0.6393810510635376 +particle/sensitivity‚0.6308444142341614 3Purity/0FRt/event_proportion‚0.3301943838596344 3Purity/1FRt/event_proportion‚0.6297432780265808 /Purity/2FRt/event_purity‚0.4285624027252197 /particle/accuracy_2_of_2‚0.6858385801315308 6validation_average_jet_accuracy‚0.3155452311038971 % _timestamp‚1.7866633299977846e+09×Ë>;åš)ß .Purity/1FRt/FRt_purity‚0.46925240755081177 )jet/accuracy_1_of_2‚0.577630341053009 particle/accuracy_1_of_1‚1 epoch‚43 trainer/global_step‚17599 _step‚1451 _runtime‚4678.527869382 0Purity/1FRt/event_purity‚0.46925240755081177 2Purity/2FRt/event_proportion‚0.040062315762043 +particle/specificity‚0.6440638303756714 'particle/f_score‚0.5539261698722839 *loss/val_total_loss‚2.5239334406896683 *jet/accuracy_1_of_1‚0.3104802668094635 (particle/accuracy‚0.6393810510635376 +particle/sensitivity‚0.6308444142341614 3Purity/0FRt/event_proportion‚0.3301943838596344 3Purity/1FRt/event_proportion‚0.6297432780265808 /Purity/2FRt/event_purity‚0.4285624027252197 /particle/accuracy_2_of_2‚0.6858385801315308 6validation_average_jet_accuracy‚0.3155452311038971 % _timestamp‚1.7866633299977846e+09 /Purity/*FRt/event_purity‚0.4669463336467743 .Purity/*FRt/FRt_purity‚0.48037365078926086 -Purity/2FRt/FRt_purity‚0.5663777589797974 /particle/accuracy_1_of_2‚0.9226521849632263 2Purity/*FRt/event_proportion‚0.669805645942688 Purity/0FRt/event_purity‚1 +jet/accuracy_2_of_2‚0.21150988340377808 +validation_accuracy‚0.21150988340377808« 5|¤–›)‚(:‹ ££ùÓÅàùà gpu.0.gpu‚64.0 gpu.0.memory‚12, gpu.0.memoryAllocated‚7.0313262939453125) gpu.0.memoryAllocatedBytes‚ 6039863296 gpu.0.temp‚37.0 gpu.0.powerWatts‚195.757' gpu.0.enforcedPowerLimitWatts‚500.0 gpu.0.powerPercent‚39.1514 gpu.0.smClock‚1410 gpu.0.memoryClock‚1593$ gpu.0.correctedMemoryErrors‚1955# gpu.0.uncorrectedMemoryErrors‚0m&Ë ˜œ)‚(: ££ùÓ³Á¦ã disk.sda2.in‚0$ memory_percent‚68.61681938171387$ disk./.usageGB‚487.1463508605957" disk.dm-0.out‚61901.8857421875' disk.nvme0c0n1.out‚61901.8857421875 disk.sda.in‚43542.96484375 disk.sda1.out‚0( proc.memory.availableMB‚ 5141.8203125# proc.memory.rssMB‚ 2354.24609375 network.sent‚ 2.2940927e+07 network.recv‚ 8.813126e+06) disk./.usagePercent‚32.69280034926801" disk.nvme0n1.in‚70043.66796875 disk.dm-0.in‚70043.66796875 disk.sda1.in‚0 disk.sda2.out‚0 cpu‚27.252684438429164% disk.nvme0n1.out‚61901.8857421875 disk.dm-1.in‚43542.96484375 disk.drbd2165.out‚0) proc.memory.percent‚14.36917781829834 proc.cpu.threads‚17 disk.dm-1.out‚ 86.578125 disk.drbd2165.in‚0 disk.sda3.in‚43542.96484375 disk.sda3.out‚ 86.578125$ disk.nvme0c0n1.in‚70043.66796875 disk.sda.out‚ 86.578125Åd æ2)- _wandbruntime‚4690  _runtime‚4690F”ðÉž)à & lr-AdamW/pg1‚5.262339081504941e-05 & lr-AdamW/pg2‚5.262339081504941e-05 trainer/global_step‚17649 % _timestamp‚1.7866633419583364e+09 _runtime‚4690.488339949 _step‚1452H v“Ο)È % _timestamp‚1.7866633419583364e+09 _runtime‚4690.488339949 _step‚1452 & lr-AdamW/pg1‚5.262339081504941e-05 & lr-AdamW/pg2‚5.262339081504941e-05 trainer/global_step‚17649¬ òñê2 )- _wandbruntime‚4690  _runtime‚4690›þÌÄ'¡)¾O @7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚5264 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚481801096953856 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 4260409251748 KBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚4876 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚ 124638879 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 5494538240 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 5368709120 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚55520000 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 4654602240 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚4287 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚486070333666816 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚48232448 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚322 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚ 124634590 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚65379335 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚486070392716800 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚481801079848960 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 4269253817856 G>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4876 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 4654602240 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚481801096953856 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚55520078 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚419967936790528 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 KDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚60 IBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚60 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 4515035007488 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚262 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚322 F?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚60 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 122905600 JBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚665 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚40706103 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚478948940939264 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚65379333 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚429 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚59255257 M@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚59049984 PCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚59049984 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17104896 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 4628521206 F>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 5494538240 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 4269295762944 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚59255257 C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚4289 D;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚5264 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚483209291803632 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚322 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚262 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 4269253817856 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚262 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚41945088 PIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚76 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚14813973 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚478948923899904 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 4260367903728 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 5368709120 JDeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4289 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚483209350191012 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚59259544 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚209 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 $ _timestamp‚1.786663341959259e+09 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚41945088 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚4287 IBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚78 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 831258624 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0À±÷'¢)ÃO E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚262 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚322 F?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚60 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 122905600 JBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚665 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚40706103 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚478948940939264 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚65379333 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚429 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚59255257 M@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚59049984 PCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚59049984 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17104896 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 4628521206 F>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 5494538240 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 4269295762944 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚59255257 C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚4289 D;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚5264 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚483209291803632 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚322 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚262 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 4269253817856 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚262 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚41945088 PIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚76 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚14813973 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚478948923899904 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 4260367903728 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 5368709120 JDeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4289 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚483209350191012 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚59259544 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚209 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 $ _timestamp‚1.786663341959259e+09 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚41945088 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚4287 IBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚78 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 831258624 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 @7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚5264 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚481801096953856 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 4260409251748 KBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚4876 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚ 124638879 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 5494538240 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 5368709120 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚55520000 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 4654602240 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚4287 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚486070333666816 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚48232448 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚322 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚ 124634590 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚65379335 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚486070392716800 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚481801079848960 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 4269253817856 G>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4876 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 4654602240 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚481801096953856 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚55520078 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚419967936790528 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 KDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.alloƒøòß»cated‚60 IBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚60 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 4515035007488­ õ…h 2£)- _wandbruntime‚4690  _runtime‚4690xþ«#Ï&¤)ÉM A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚4295 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚419990984163328 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 125829120 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17104896 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 5494538240 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚322 trainer/global_step‚17649 % _timestamp‚1.7866633421204748e+09 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚223 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 4260639824408 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚262 D=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚60 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚4293 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚14814792 B5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5264 SFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚66776064 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521206 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 5368709120 I@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚4876 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚65382888 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚478975126732800 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚481827402186752 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 4269484886528 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚55523037 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 4517085184 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚20643840 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚486096946126336 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚59262837 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚65382888 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 122905600 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚40708310 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚481827402186752 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚322 K>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59053056 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚486096946126336 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654602240 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚424506264529408 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚46132224 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 4260598476340 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 5494538240 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 4552982528 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 122905600 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 125829120 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 4269526834688 H@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚665 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚4293 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚483235766557208 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚41348068 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚322 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚61478912 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚65382886 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 NADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚58387428 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚ 124645725 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚40708308 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚65382886 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚481827419291648 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚14814729 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 4552982528 NGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚63 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 4269484886528 G@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚65 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 831258624 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 4515280366080 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 E8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 5494538240 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 125829120 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 4269526834688 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚419991004807168 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 4515326498304 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚ 124641430 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚424506331305472 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 4654602240 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 122306526 G@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚60 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 124645725 EDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚41948160 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚322 _runtime‚4690.6507694724_4öÔ&¥)ÎM NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚65382886 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 NADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚58387428 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚ 124645725 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚40708308 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚65382886 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚481827419291648 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚14814729 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 4552982528 NGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚63 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 4269484886528 G@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚65 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 831258624 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 4515280366080 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 E8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 5494538240 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 125829120 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 4269526834688 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚419991004807168 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 4515326498304 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚ 124641430 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚424506331305472 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 4654602240 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 122306526 G@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚60 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 124645725 EDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚41948160 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚322 _runtime‚4690.650769472 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚4295 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚419990984163328 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 125829120 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17104896 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 5494538240 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚322 trainer/global_step‚17649 % _timestamp‚1.7866633421204748e+09 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚223 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 4260639824408 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚262 D=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚60 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚4293 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚14814792 B5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5264 SFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚66776064 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521206 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 5368709120 I@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚4876 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚65382888 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚478975126732800 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚481827402186752 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 4269484886528 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚55523037 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 4517085184 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚20643840 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚486096946126336 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚59262837 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚65382888 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 122905600 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚40708310 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚481827402186752 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚322 K>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59053056 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚486096946126336 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654602240 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚424506264529408 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚46132224 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 4260598476340 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 5494538240 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 4552982528 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 122905600 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 125829120 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 4269526834688 H@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚665 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚4293 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚483235766557208 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚41348068 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚322 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚61478912 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0® iÃ2¦)- _wandbruntime‚4690  _runtime‚4690ôz-t§)î $ _timestamp‚1.786663342121378e+09 0loss/FRt1/assignment_loss‚0.4602191377566411 0loss/FRt2/detection_loss‚0.48753780711865147 epoch‚44 _runtime‚4690.651186551 _step‚1455 /loss/FRt2/assignment_loss‚0.355705995825117 0loss/FRt1/detection_loss‚0.47079482834658243 %loss/total_loss‚1.774257769046992 trainer/global_step‚17649Þ å^y¨)ó 0loss/FRt1/assignment_loss‚0.4602191377566411 0loss/FRt2/detection_loss‚0.48753780711865147 epoch‚44 _runtime‚4690.651186551 _step‚1455 /loss/FRt2/assignment_loss‚0.355705995825117 0loss/FRt1/detection_loss‚0.47079482834658243 %loss/total_loss‚1.774257769046992 trainer/global_step‚17649 $ _timestamp‚1.786663342121378e+09¯ áéò•©)‚(:Š ²£ùÓû¯¤á gpu.0.gpu‚51.0 gpu.0.memory‚9, gpu.0.memoryAllocated‚7.0313262939453125) gpu.0.memoryAllocatedBytes‚ 6039863296 gpu.0.temp‚35.0 gpu.0.powerWatts‚221.473' gpu.0.enforcedPowerLimitWatts‚500.0 gpu.0.powerPercent‚44.2946 gpu.0.smClock‚1410 gpu.0.memoryClock‚1593$ gpu.0.correctedMemoryErrors‚1955# gpu.0.uncorrectedMemoryErrors‚0»×ÃΡª)‚(:– ²£ùÓó°ùã( disk.nvme0c0n1.out‚61906.85888671875) proc.memory.availableMB‚ 5141.55078125# proc.memory.rssMB‚ 2354.24609375 disk.sda.out‚ 86.9140625 disk.sda1.in‚0 disk.drbd2165.in‚0) proc.memory.percent‚14.36917781829834 cpu‚27.252404063702237 network.sent‚ 2.3008835e+07* disk./.usagePercent‚32.692809821546774 disk.sda2.in‚0" disk.nvme0n1.in‚70043.89453125 disk.dm-1.out‚ 86.9140625 disk.sda3.in‚43542.96484375 disk.sda3.out‚ 86.9140625$ disk.nvme0c0n1.in‚70043.89453125 network.recv‚ 8.835532e+06% disk./.usageGB‚487.14649200439453 disk.dm-1.in‚43542.96484375 disk.drbd2165.out‚0 disk.dm-0.in‚70043.89453125$ memory_percent‚68.61846446990967 proc.cpu.threads‚17 disk.sda.in‚43542.96484375 disk.sda1.out‚0 disk.sda2.out‚0& disk.nvme0n1.out‚61906.85888671875# disk.dm-0.out‚61906.85888671875ÀOX2«)- _wandbruntime‚4702  _runtime‚4702Žqlˬ)Å trainer/global_step‚17699 % _timestamp‚1.7866633544506779e+09 _runtime‚4702.982914501 _step‚1456 ' lr-AdamW/pg1‚5.0434019556642925e-05 ' lr-AdamW/pg2‚5.0434019556642925e-05}ðgÅЭ)Ê _runtime‚4702.982914501 _step‚1456 ' lr-AdamW/pg1‚5.0434019556642925e-05 ' lr-AdamW/pg2‚5.0434019556642925e-05 trainer/global_step‚17699 % _timestamp‚1.7866633544506779e+09° Yä‡2®)- _wandbruntime‚4702  _runtime‚4702"~sº&¯)¿O LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 5368709120 G>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4876 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 4654602240 PCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚59049984 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17104896 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚41348020 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 4628521206 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚480258230616064 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 4517085184 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚322 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 5494538240 JDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 JBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚665 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚40816353 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚425647686501376 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚483117196738560 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 4280882528768 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 4280882528768 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚55671208 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚487398096372224 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚487398096372224 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚48232448 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 5494538240 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 845178880 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚65556983 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚483117213843456 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚14854853 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚65556985 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚4287 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚ 124981631 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 4552982528 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚41945088 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚20643840 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚322 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚322 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚55671129 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 831258624 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 4527360427008 KDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚60 % _timestamp‚1.7866633544513583e+09 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 _step‚1457 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚487398037322240 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚262 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 4654602240 C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚4289 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚483117213843456 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚14854776 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 122306526 F?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚60 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚483117196738560 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚429 PIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚77 trainer/global_step‚17699 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚ 124977342 JDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 M@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚59049984 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 122905600 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚4287 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚61478912 BDeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4289 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚209 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 4527312194560 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 4271971062356 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pooæC€ l.peak‚262 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚65556983 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚59424646 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚484530201678420deÖÊ'°)ÄO M@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚59049984 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 122905600 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚4287 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚61478912 BDeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4289 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚209 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 4527312194560 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 4271971062356 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚262 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚65556983 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚59424646 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚484530201678420 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 5368709120 G>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4876 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 4654602240 PCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚59049984 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17104896 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚41348020 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 4628521206 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚480258230616064 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 4517085184 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚322 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 5494538240 JDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 JBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚665 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚40816353 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚425647686501376 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚483117196738560 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 4280882528768 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 4280882528768 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚55671208 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚487398096372224 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚487398096372224 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚48232448 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 5494538240 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 845178880 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚65556983 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚483117213843456 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚14854853 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚65556985 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚4287 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚ 124981631 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 4552982528 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚41945088 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚20643840 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚322 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚322 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚55671129 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 831258624 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 4527360427008 KDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚60 % _timestamp‚1.7866633544513583e+09 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 _step‚1457 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚487398037322240 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚262 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 4654602240 C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚4289 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚483117213843456 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚14854776 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 122306526 F?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚60 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚483117196738560 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚429 PIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚77 trainer/global_step‚17699 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚ 124977342 JDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0± É"¶¶2±)- _wandbruntime‚4703  _runtime‚4703ü[¶]Ï&²)ÉM GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚262 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚59427939 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚209 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 G@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚60 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 4654602240 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521206 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚44035072 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚65560538 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚425670978623488 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 845178880 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 4527605176320 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚ 124988477 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚483143536181248 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 124988477 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚40818560 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚480284416409600 % _timestamp‚1.7866633546164281e+09 G@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚72 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚223 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚421143352803328 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 4527561141248 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 4272201635016 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚ 124984182 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17104896 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚4293 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 >5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5264 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚4293 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 831258624 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 122306526 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 125829120 P8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 4552982528 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 122905600 I@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚4876 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚55674263 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚40818558 NADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚58387428 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚41348068 _step‚1458 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚487424649781760 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚41948160 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚487424590728704 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 4281113600512 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚59427939 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 4281071652352 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 122905600 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚ 124984182 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚4295 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚65560536 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚55674191 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚484556559657188 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17104896 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚425670913944576 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚65560536 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚59423646 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 5368709120 K>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59053056 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚65560538 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 5494538240 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚487424649781760 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚322 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚322 trainer/global_step‚17699 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 4281071652352 B9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚5264 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚429 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 5368709120 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654602240 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚421143373447168 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 4272160286948 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 5494538240 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 125829120 NADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚59053056 SFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚64678912 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 5368709120 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 4552982528 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚20643840 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚484556618044616 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 125829120 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚262 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚59423646 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚322 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚483143519076352 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 4517085184 IBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚60 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚480284399370240 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 5494538240 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 D=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚60 _runtime‚4703.146709707új×aÔ&³)ÎM M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521206 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚44035072 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚65560538 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚425670978623488 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 845178880 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 4527605176320 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚ 124988477 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚483143536181248 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 124988477 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚40818560 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚480284416409600 % _timestamp‚1.7866633546164281e+09 G@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚72 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚223 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚421143352803328 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 4527561141248 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 4272201635016 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚ 124984182 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17104896 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚4293 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 >5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5264 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚4293 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 831258624 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 122306526 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 125829120 P8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 4552982528 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 122905600 I@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚4876 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚55674263 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚40818558 NADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚58387428 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚41348068 _step‚1458 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚487424649781760 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚41948160 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚487424590728704 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 4281113600512 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚59427939 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 4281071652352 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 122905600 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚ 124984182 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚4295 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚65560536 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚55674191 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚484556559657188 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17104896 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚425670913944576 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚65560536 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚59423646 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 5368709120 K>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59053056 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚65560538 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 5494538240 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚487424649781760 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚322 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚322 trainer/global_step‚17699 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 4281071652352 B9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚5264 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚429 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 5368709120 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654602240 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚421143373447168 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 4272160286948 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 5494538240 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 125829120 NADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚59053056 SFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚64678912 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 5368709120 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 4552982528 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚20643840 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚484556618044616 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 125829120 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚262 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚59423646 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚322 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚483143519076352 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 4517085184 IBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚60 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚480284399370240 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 5494538240 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 D=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚60 _runtime‚4703.146709707 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚262 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚59427939 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚209 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 G@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚60 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 4654602240² U»i2´)- _wandbruntime‚4703  _runtime‚4703*¯­“uµ)ï 1loss/FRt2/assignment_loss‚0.36722527684622625 %loss/total_loss‚1.836680308514497 trainer/global_step‚17699 % _timestamp‚1.7866633546177626e+09 _runtime‚4703.147534876 1loss/FRt1/assignment_loss‚0.48359017126158055 /loss/FRt1/detection_loss‚0.5036984753011862 .loss/FRt2/detection_loss‚0.482166385105504 epoch‚44 _step‚1459‰×®z¶)ô epoch‚44 _step‚1459 1loss/FRt2/assignment_loss‚0.36722527684622625 %loss/total_loss‚1.836680308514497 trainer/global_step‚17699 % _timestamp‚1.7866633546177626e+09 _runtime‚4703.147534876 1loss/FRt1/assignment_loss‚0.48359017126158055 /loss/FRt1/detection_loss‚0.5036984753011862 .loss/FRt2/detection_loss‚0.482166385105504³ ¹M’G¡·)‚(:– Á£ùÓø„«á gpu.0.gpu‚62.0 gpu.0.memory‚11, gpu.0.memoryAllocated‚7.0313262939453125) gpu.0.memoryAllocatedBytes‚ 6039863296 gpu.0.temp‚36.0 gpu.0.powerWatts‚123.175' gpu.0.enforcedPowerLimitWatts‚500.0) gpu.0.powerPercent‚24.634999999999998 gpu.0.smClock‚1410 gpu.0.memoryClock‚1593$ gpu.0.correctedMemoryErrors‚1955# gpu.0.uncorrectedMemoryErrors‚0Š@Ó ¸)‚(:• Á£ùÓ­“ðâ network.sent‚ 2.3083994e+07( disk.nvme0c0n1.out‚62136.98486328125( proc.memory.availableMB‚ 5141.7578125) proc.memory.percent‚14.36917781829834 cpu‚27.251935182939935 proc.cpu.threads‚17% disk./.usageGB‚487.64662170410156 disk.sda3.out‚ 87.14453125 disk.sda.in‚43542.96484375 disk.sda.out‚ 87.14453125 disk.sda1.in‚0 disk.sda2.in‚0! disk.nvme0n1.in‚ 70046.2421875 disk.dm-1.out‚ 87.14453125 network.recv‚ 8.868206e+06* disk./.usagePercent‚32.726373945330884 disk.sda3.in‚43542.96484375# disk.nvme0c0n1.in‚ 70046.2421875 disk.sda1.out‚0& disk.nvme0n1.out‚62136.98486328125 disk.dm-1.in‚43542.96484375 disk.drbd2165.out‚0 disk.sda2.out‚0 disk.drbd2165.in‚0 disk.dm-0.in‚ 70046.2421875# disk.dm-0.out‚62136.98486328125$ memory_percent‚68.61720085144043# proc.memory.rssMB‚ 2354.24609375èÅÐü2¹)- _wandbruntime‚4715  _runtime‚4715[—kÚnº)Å ' lr-AdamW/pg2‚4.8289580113914545e-05 trainer/global_step‚17749 % _timestamp‚1.7866633666094ÜJÌ]425e+09 _runtime‚4715.139431921 _step‚1460 ' lr-AdamW/pg1‚4.8289580113914545e-05-CÀл)Ê ' lr-AdamW/pg1‚4.8289580113914545e-05 ' lr-AdamW/pg2‚4.8289580113914545e-05 trainer/global_step‚17749 % _timestamp‚1.7866633666094425e+09 _runtime‚4715.139431921 _step‚1460´ t\{#2¼)- _wandbruntime‚4715  _runtime‚4715ÑDhÄ'½)¾O D;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚5264 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚422272674070528 IBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚60 F?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚60 @7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚5264 F>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 122905600 JBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚665 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚422272694714368 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 4283491524944 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚65734635 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚61478912 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚481567520292864 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 4283532872964 C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚4289 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚429 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 4552982528 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚4287 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚59585461 IBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚73 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 5368709120 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚41945088 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 5494538240 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 4654602240 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚488725740977664 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚59589748 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚322 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚262 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚262 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚65734635 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚65734633 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 4292427349504 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 5368709120 JDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4876 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚484433330733056 PCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚59049984 PIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚71 _runtime‚4715.143872198 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 4292427349504 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚41348020 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 5368709120 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚485850994778448 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 4517085184 $ _timestamp‚1.786663366610389e+09 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚484433313628160 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚ 125324383 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 831258624 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚50329600 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 4539580917248 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 KBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚4876 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 4628521206 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚322 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17104896 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚14895949 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚209 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 845178880 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 125829120 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 4292469294592 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚484433330733056 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17104896 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚485851053165828 KDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚60 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 122905600 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚262 _step‚1461 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚ 125320094 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚488725740977664 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4289 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚55822481 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚40926603 UHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚70973440 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 PCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚58387380 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚59585461 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚41945088 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚223 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚481567503253504 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚322 JDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 122905600 JBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚665 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚422272694714368 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 4283491524944 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚65734635 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚61478912 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚481567520292864 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 4283532872964 C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚4289 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚429 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 4552982528 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚4287 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚59585461 IBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚73 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 5368709120 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚41945088 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 5494538240 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 4654602240 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚488725740977664 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚59589748 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚322 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚262 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚262 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚65734635 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚65734633 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 4292427349504 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 5368709120 JDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4876 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚484433330733056 PCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚59049984 PIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚71 _runtime‚4715.143872198 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 4292427349504 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚41348020 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 5368709120 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚485850994778448 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 4517085184 $ _timestamp‚1.786663366610389e+09 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚484433313628160 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚ 125324383 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 831258624 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚50329600 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 4539580917248 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 KBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚4876 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 4628521206 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚322 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17104896 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚14895949 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚209 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 845178880 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 125829120 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 4292469294592 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚484433330733056 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17104896 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚485851053165828 KDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚60 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 122905600 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚262 _step‚1461 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚ 125320094 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚488725740977664 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4289 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚55822481 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚40926603 UHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚70973440 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 PCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚58387380 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚59585461 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚41945088 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚223 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚481567503253504 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚322 JDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚55825506 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 5494538240 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚41948160 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 4292700366336 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 122905600 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚426835549488640 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚262 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚59588748 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚55825573 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚40928808 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚59588748 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚488752294384128 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚484459653070848 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚48229376 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚4295 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚4293 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 125331229 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚209 NADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚58387428 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 EDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚481593689047040 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚322 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚322 _step‚1462 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚488752353437184 DDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59053056 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚422295742087168 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17104896 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚20643840 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521206 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚484459635965952 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚41948160 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚322 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚485877411144596 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚65738188 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654602240 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚14896763 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚262 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚223 >5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5264 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚484459635965952 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚ 125326934 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚429 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 122306526 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 4552982528 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 4292658418176 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚426835618361856 SFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚68873216 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 125829120 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 125829120 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 831258624 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 _runtime‚4715.302824042 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 4654602240 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚59593041 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 4283763445624 IBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚60 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚ 125326934 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 845178880 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 4539876274688 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 G@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚60 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚65738186 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚422295721443328 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚61478912 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 4517085184 G@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚67 NGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚65 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚ 125331229 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 4283722097556 D=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚60 B9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚5264 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚322 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 5368709120Õ«ÙÜÁ)ÍM SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 122905600 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚426835549488640 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚262 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚59588748 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚55825573 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚40928808 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚59588748 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚488752294384128 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚484459653070848 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚48229376 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚4295 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚4293 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 125331229 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚209 NADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚58387428 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 EDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚481593689047040 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚322 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚322 _step‚1462 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚488752353437184 DDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59053056 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚422295742087168 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17104896 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚20643840 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521206 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚484459635965952 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚41948160 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚322 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚485877411144596 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚65738188 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654602240 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚14896763 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚262 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚223 >5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5264 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚484459635965952 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚ 125326934 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚429 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 122306526 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 4552982528 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 4292658418176 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚426835618361856 SFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚68873216 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 125829120 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 125829120 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 831258624 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 _runtime‚4715.302824042 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 4654602240 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚59593041 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 4283763445624 IBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚60 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚ 125326934 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 845178880 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 4539876274688 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 G@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚60 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚65738186 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚422295721443328 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚61478912 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 4517085184 G@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚67 NGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚65 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚ 125331229 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 4283722097556 D=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚60 B9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚5264 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚322 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 5368709120 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 4292658418176 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚55825506 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 5494538240 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚41948160 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 4292700366336¶ sÆsj2Â)- _wandbruntime‚4715  _runtime‚4715„.FwÃ)ñ epoch‚44 % _timestamp‚1.7866633667734702e+09 _runtime‚4715.303271336 _step‚1463 0loss/FRt1/detection_loss‚0.48026265167431353 /loss/FRt2/detection_loss‚0.4816816655230664 &loss/total_loss‚1.8372113869965863 trainer/global_step‚17749 0loss/FRt1/assignment_loss‚0.5012148991293208 1loss/FRt2/assignment_loss‚0.37405217066988555}Ìþ€|Ä)ö 0loss/FRt1/assignment_loss‚0.5012148991293208 1loss/FRt2/assignment_loss‚0.37405217066988555 epoch‚44 % _timestamp‚1.7866633667734702e+09 _runtime‚4715.303271336 _step‚1463 0loss/FRt1/detection_loss‚0.48026265167431353 /loss/FRt2/detection_loss‚0.4816816655230664 &loss/total_loss‚1.8372113869965863 trainer/global_step‚17749· ´Ô% –Å)‚(:‹ УùÓ’€ýà gpu.0.gpu‚64.0 gpu.0.memory‚12, gpu.0.memoryAllocated‚7.0313262939453125) gpu.0.memoryAllocatedBytes‚ 6039863296 gpu.0.temp‚36.0 gpu.0.powerWatts‚212.601' gpu.0.enforcedPowerLimitWatts‚500.0 gpu.0.powerPercent‚42.5202 gpu.0.smClock‚1410 gpu.0.memoryClock‚1593$ gpu.0.correctedMemoryErrors‚1955# gpu.0.uncorrectedMemoryErrors‚0È8½–Æ)‚(:‹ УùÓÿÍÎâ disk.dm-1.out‚87.40625 disk.drbd2165.in‚0" disk.dm-0.out‚62246.9326171875 disk.sda3.in‚43542.96484375 disk.sda2.in‚0 disk.sda2.out‚0% disk./.usageGB‚487.64049530029297 disk.sda.out‚87.40625 disk.sda1.out‚0" disk.nvme0n1.in‚70047.08984375$ memory_percent‚68.61882209777832( proc.memory.availableMB‚ 5141.4921875# proc.memory.rssMB‚ 2354.24609375) proc.memory.percent‚14.36917781829834 network.recv‚ 8.900598e+06 disk.dm-1.in‚43542.96484375 disk.drbd2165.out‚0 disk.sda3.out‚87.40625' disk.nvme0c0n1.out‚62246.9326171875 disk.sda.in‚43542.96484375 cpu‚27.251512260463148 proc.cpu.threads‚17 disk.dm-0.in‚70047.08984375$ disk.nvme0c0n1.in‚70047.08984375 disk.sda1.in‚0% disk.nvme0n1.out‚62246.9326171875 network.sent‚ 2.3159071e+07) disk./.usagePercent‚32.72596279723089o™+Á2Ç)- _wandbruntime‚4727  _runtime‚4727º‘ª ÉÈ)à & lr-AdamW/pg2‚4.619021022020275e-05 trainer/global_step‚17799 % _timestamp‚1.7866633787334933e+09 _runtime‚4727.263514197 _step‚1464 & lr-AdamW/pg1‚4.619021022020275e-05Ú8‚yÎÉ)È & lr-AdamW/pg1‚4.619021022020275e-05 & lr-AdamW/pg2‚4.619021022020275e-05 trainer/global_step‚17799 % _timestamp‚1.7866633787334933e+09 _runtime‚4727.263514197 _step‚1464¸ ímg¿2Ê)- _wandbruntime‚4727  _runtime‚4727þµ[Å'Ë)¿O \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚485749430517760 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 4628521206 IBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚60 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚429 F>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚55973305 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚14936452 UHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚60487680 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 5368709120 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚ 125662846 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 4295053335552 M@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚59049984 JBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚665 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚485749447622656 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚41036853 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚39843840 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 4517085184 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚41348020 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 125829120 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚4287 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 5368709120 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚65912283 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 4654602240 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚487171904653236 _runtime‚4727.267866093 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚322 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 125829120 @7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚5264 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚490053503683072 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17104896 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 4552982528 KBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚4876 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚322 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 trainer/global_step‚17799 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚59754850 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚41945088 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 4295094683572 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚485749447622656 IBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚65 PIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚63 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 125829120 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚322 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚427976900209152 BDeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4289 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚482876792930304 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚61478912 % _timestamp‚1.7866633787344272e+09 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 5494538240 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚59750563 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚490053444633088 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 4552982528 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚20643840 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 122306526 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚ 125667135 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚262 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚485749430517760 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚65912283 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚59754850 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 KDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚60 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚41036855 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚41945088 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚14936515 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 122905600 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚427976960696832 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 4551897342464 F?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚60 PCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚59049984 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 5494538240 G>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4876 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 4304056060416 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚65912285 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚490053503683072 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚490053444633088 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚223 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 PCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚58387380 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚262 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚262 D;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚5264 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚487171846265856 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚322 _step‚1465 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚4287 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 5368709120 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0`gÕ–Ê'Ì)ÄO PIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚63 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 125829120 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚322 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚427976900209152 BDeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4289 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚482876792930304 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚61478912 % _timestamp‚1.7866633787344272e+09 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 5494538240 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚59750563 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚490053444633088 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 4552982528 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚20643840 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 122306526 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚ 125667135 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚262 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚485749430517760 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚65912283 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚59754850 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 KDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚60 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚41036855 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚41945088 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚14936515 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 122905600 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚427976960696832 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 4551897342464 F?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚60 PCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚59049984 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 5494538240 G>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4876 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 4304056060416 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚65912285 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚490053503683072 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚490053444633088 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚223 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 PCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚58387380 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚262 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚262 D;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚5264 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚487171846265856 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚322 _step‚1465 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚4287 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 5368709120 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚485749430517760 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 4628521206 IBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚60 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚429 F>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚55973305 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚14936452 UHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚60487680 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 5368709120 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚ 125662846 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 4295053335552 M@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚59049984 JBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚665 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚485749447622656 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚41036853 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚39843840 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 4517085184 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚41348020 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 125829120 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚4287 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 5368709120 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚65912283 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 4654602240 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚487171904653236 _runtime‚4727.267866093 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚322 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 125829120 @7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚5264 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚490053503683072 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17104896 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 4552982528 KBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚4876 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚322 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 trainer/global_step‚17799 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚59754850 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚41945088 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 4295094683572 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚485749447622656 IBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚65¹ ¬ ¥2Í)- _wandbruntime‚4727  _runtime‚4727éššÎ)ÉM ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚485775752855552 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 122306526 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚482902995763200 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚ 125669686 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚59758143 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 125673981 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 5494538240 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚262 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 BDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59053056 PDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚ 125673981 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚485775752855552 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚41948160 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚55976329 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚61478912 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 5368709120 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚322 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚4293 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚485775769960448 I@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚4876 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 5368709120 IBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚60 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 4552982528 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 4304287132160 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚59758143 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚55976400 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚490079998039552 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚65915838 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚423448090083328 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚48229376 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚487198262632004 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚490080057092608 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654602240 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521206 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚4295 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17104896 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 4304245184000 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚429 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚41039058 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 >5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5264 G@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚71 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 845178880 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 4295283908164 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚322 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚322 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚262 _step‚1466 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚59753850 B9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚5264 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 4517085184 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚322 trainer/global_step‚17799 NADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚59053056 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚262 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 EDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚485775769960448 D=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚60^쌂Ô&Ï)ÎM [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚485775769960448 D=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚60 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚485775752855552 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 122306526 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚482902995763200 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚ 125669686 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚59758143 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 125673981 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 5494538240 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚262 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 BDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59053056 PDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚ 125673981 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚485775752855552 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚41948160 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚55976329 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚61478912 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 5368709120 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚322 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚4293 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚485775769960448 I@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚4876 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 5368709120 IBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚60 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 4552982528 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 4304287132160 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚59758143 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚55976400 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚490079998039552 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚65915838 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚423448090083328 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚48229376 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚487198262632004 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚490080057092608 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654602240 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521206 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚4295 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17104896 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 4304245184000 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚429 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚41039058 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 >5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5264 G@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚71 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 845178880 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 4295283908164 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚322 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚322 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚262 _step‚1466 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚59753850 B9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚5264 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 4517085184 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚322 trainer/global_step‚17799 NADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚59053056 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚262 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 EDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0º é„þC2Ð)- _wandbruntime‚4727  _runtime‚4727ø™ULvÑ)ð &loss/total_loss‚1.7611105633975006 % _timestamp‚1.7866633788976862e+09 _step‚1467 1loss/FRt2/assignment_loss‚0.34881122651268526 epoch‚44 trainer/global_step‚17799 _runtime‚4727.427586174 0loss/FRt1/assignment_loss‚0.4826055878938102 0loss/FRt1/detection_loss‚0.46043874944405927 .loss/FRt2/detection_loss‚0.469254999546946 ª® {Ò)õ 1loss/FRt2/assignment_loss‚0.34881122651268526 epoch‚44 trainer/global_step‚17799 _runtime‚4727.427586174 0loss/FRt1/assignment_loss‚0.4826055878938102 0loss/FRt1/detection_loss‚0.46043874944405927 .loss/FRt2/detection_loss‚0.469254999546946 &loss/total_loss‚1.7611105633975006 % _timestamp‚1.7866633788976862e+09 _step‚1467» "'æq2Ó)- _wandbruntime‚4739  _runtime‚4739‡Q®™ËÔ)Å trainer/global_step‚17849 % _timestamp‚1.7866633911432414e+09 _runtime‚4739.673260841 _step‚1468 ' lr-AdamW/pg1‚4.4136044714112586e-05 ' lr-AdamW/pg2‚4.4136044714112586e-05¬õ«êÐÕ)Ê % _timestamp‚1.7866633911432414e+09 _runtime‚4739.673260841 _step‚1468 ' lr-AdamW/pg1‚4.4136044714112586e-05 ' lr-AdamW/pg2‚4.4136044714112586e-05 trainer/global_step‚17849¼ ¾¾M®2Ö)- _wandbruntime‚4739  _runtime‚4739{9q­Ä'×)¾O YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚487065547407360 PCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚59049984 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 4564148612608 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚491381207338496 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 4306656494180 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 5494538240 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 125829120 F>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚488492756140644 trainer/global_step‚17849 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚4287 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 4654602240 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 KDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚60 _runtime‚ 4739.67789391 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚ 126005598 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚59919952 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚59915665 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 4552982528 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚429141630936576 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 5368709120 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚41945088 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 122905600 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚209 M@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚59049984 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 4315642826240 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4289 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚4287 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚59915665 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 5368709120 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 125829120 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚322 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 _step‚1469 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚56124560 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚223 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚50329600 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 5368709120 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚66089933 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚487065547407360 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚322 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚491381148288512 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 4654602240 C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚4289 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚66089933 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚429 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚14977388 BDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚491381148288512 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚56124491 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚429141559963136 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 JDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4876 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚41945088 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚487065564512256 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17104896 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚59919952 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 125829120 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚20643840 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 831258624 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚262 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚41147105 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 4628521206 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚322m"´Ä Ø)ÃO F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚491381148288512 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 4654602240 C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚4289 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚66089933 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚429 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚14977388 BDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚491381148288512 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚56124491 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚429141559963136 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 JDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4876 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚41945088 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚487065564512256 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17104896 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚59919952 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 125829120 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚20643840 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 831258624 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚262 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚41147105 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 4628521206 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚322 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚487065547407360 PCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚59049984 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 4564148612608 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚491381207338496 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 4306656494180 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 5494538240 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 125829120 F>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚488492756140644 trainer/global_step‚17849 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚4287 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 4654602240 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 KDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚60 _runtime‚ 4739.67789391 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚ 126005598 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚59919952 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚59915665 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 4552982528 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚429141630936576 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 5368709120 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚41945088 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 122905600 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚209 M@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚59049984 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 4315642826240 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4289 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚4287 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚59915665 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 5368709120 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 125829120 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚322 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 _step‚1469 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚56124560 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚223 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚50329600 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 5368709120 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚66089933 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚487065547407360 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚322 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0½ [À2Ù)- _wandbruntime‚4739  _runtime‚4739@^æSÏ&Ú)ÉM EDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚56127471 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚41149308 SFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚68873216 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚262 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚66093486 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚4295 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 5494538240 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚59923245 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17104896 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚41149310 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 K>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59053056 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚14978163 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚61478912 >5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5264 DDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 122905600 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 5368709120 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚262 trainer/global_step‚17849 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚487091869745152 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 4315873897984 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚56127543 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚48229376 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚4293 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚429164851645952 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 5368709120 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 125829120 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 I@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚4876 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 5494538240 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚59918952 P8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 5494538240 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚ 126012438 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 4552982528 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚491407760748032 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚209 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 4564392922624 BDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654602240 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚66093486 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚59918952 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 831258624 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚41348068 IBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚60 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚424600458723328 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 NADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚58387428 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚322 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 4552982528 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 126016733 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚ 126016733 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚429164920519168 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521206 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 4306845718772 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 122306526 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 125829120 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚322 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚322 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚41948160 NGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚70 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚488519114119412 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚484212285440000 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 125829120 D=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚60 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚322 NADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚59053056 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚491407701694976 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚ 126012438 _step‚1470 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚41948160·L—ïÔ&Û)ÎM [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚487091886850048 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 4315831949824 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚487091869745152 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 845178880 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚20643840 E8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 5494538240 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚ 126012438 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 4552982528 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚491407760748032 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚209 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 4564392922624 BDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654602240 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚66093486 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚59918952 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 831258624 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚41348068 IBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚60 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚424600458723328 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 NADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚58387428 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚322 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 4552982528 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 126016733 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚ 126016733 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚429164920519168 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521206 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 4306845718772 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 122306526 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 125829120 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚322 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚322 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚41948160 NGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚70 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚488519114119412 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚484212285440000 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 125829120 D=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚60 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚322 NADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚59053056 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚491407701694976 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚ 126012438 _step‚1470 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚41948160 EDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚56127471 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚41149308 SFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚68873216 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚262 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚66093486 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚4295 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 5494538240 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚59923245 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17104896 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚41149310 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 K>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59053056 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚14978163 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚61478912 >5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5264 DDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 122905600 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 5368709120 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚262 trainer/global_step‚17849 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚487091869745152 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 4315873897984 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚56127543 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚48229376 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚4293 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚429164851645952 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 5368709120 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 125829120 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 I@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚4876 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 5494538240 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚59918952 PDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 @7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚5264 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚430306262694400 PCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚58387380 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚485495372283904 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17104896 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚488381664296960 PIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚57 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚15018055 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚209 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚492708851943936 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚66267583 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚425729800634368 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 4327187646976 KBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚4876 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚322 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 _runtime‚4751.899060282 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 4318218304788 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 122306526 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚262 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚66267585 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 4654602240 JDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚492708851943936 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚66267585 JBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚665 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚56275410 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 _step‚1473 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 122905600 IBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚60 JDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 5494538240 KDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚60 G>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4876 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚492708910993920 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚56275351 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚41940992 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚41348020 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 125829120 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 F?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚60 trainer/global_step‚17899 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 4552982528 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4289 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚322 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚262 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 IBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚59 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚20643840 BDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 @7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚5264 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚430306262694400 PCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚58387380 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚485495372283904 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17104896 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚488381664296960 PIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚57 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚15018055 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚209 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚492708851943936 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚66267583 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚425729800634368 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 4327187646976 KBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚4876 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚322 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 _runtime‚4751.899060282 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 4318218304788 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 122306526 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚262 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚66267585 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 4654602240 JDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚492708851943936 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚66267585 JBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚665 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚56275410 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 _step‚1473 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 122905600 IBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚60 JDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 5494538240 KDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚60 G>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4876 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚492708910993920 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚56275351 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚41940992 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚41348020 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 125829120 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 F?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚60 trainer/global_step‚17899 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 4552982528 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4289 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚322 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚262 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 IBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚59 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚20643840 BDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59053056 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚425752827363328 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚66271138 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚262 IBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚60 _step‚1474 NADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚59053056 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654602240 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 122905600 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 845178880 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 NADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚58387428 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚60084054 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17104896 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚489839965606820 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚485521558077440 >5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5264 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚15018873 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚262 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 4327460663808 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚60088347 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚209 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521206 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚4293 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 125829120 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚322 EDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 126359485 I@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚4876 _runtime‚4752.061632997 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 4552982528 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 122905600 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚60084054 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚20643840 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚485521575116800 $ _timestamp‚1.786663403531397e+09 PDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚488407986634752 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 5494538240 trainer/global_step‚17899 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚ 126359485 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 4327418715648 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17104896 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 4327460663808 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚66271136 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚56278368 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 4654602240 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚488408003739648 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚ 126355190 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚41259558 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚15018810 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚41948160 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚61478912 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚489840023994248 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚ 126355190 B9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚5264 SFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚64678912 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 4318448877448 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚41348068 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 5368709120 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 5368709120 D=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚60 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 4576709512704 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 831258624 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 4576665477632 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚4295 E8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 B6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚322 G@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚65 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚425752848007168 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 125829120 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚322 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 H@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚665 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚41259560oÏí"¶é)ÍM D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚262 IBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚60 _step‚1474 NADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚59053056 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654602240 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 122905600 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 845178880 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 NADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚58387428 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚60084054 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17104896 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚489839965606820 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚485521558077440 >5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5264 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚15018873 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚262 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 4327460663808 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚60088347 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚209 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521206 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚4293 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 125829120 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚322 EDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 126359485 I@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚4876 _runtime‚4752.061632997 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 4552982528 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 122905600 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚60084054 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚20643840 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚485521575116800 $ _timestamp‚1.786663403531397e+09 PDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚488407986634752 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 5494538240 trainer/global_step‚17899 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚ 126359485 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 4327418715648 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17104896 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 4327460663808 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚66271136 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚56278368 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 4654602240 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚488408003739648 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚ 126355190 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚41259558 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚15018810 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚41948160 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚61478912 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚489840023994248 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚ 126355190 B9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚5264 SFDeviceStatsM˜Gtonitor.on_train_batch_end/inactive_split_bytes.all.current‚64678912 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 4318448877448 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚41348068 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 5368709120 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 5368709120 D=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚60 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 4576709512704 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 831258624 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 4576665477632 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚4295 E8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 B6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚322 G@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚65 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚425752848007168 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 125829120 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚322 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 H@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚665 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚41259560 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚223 K>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59053056 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚425752827363328 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚66271138 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 Yç/(2ê)- _wandbruntime‚4752  _runtime‚4752hß}të)î 0loss/FRt2/assignment_loss‚0.3720351735573884 /loss/FRt1/detection_loss‚0.4696548283524617 .loss/FRt2/detection_loss‚0.462416619733817 trainer/global_step‚17899 _runtime‚4752.062251496 0loss/FRt1/assignment_loss‚0.4824876080707243 &loss/total_loss‚1.7865942297143915 epoch‚44 % _timestamp‚1.7866634035324266e+09 _step‚1475ÂÕyì)ó % _timestamp‚1.7866634035324266e+09 _step‚1475 0loss/FRt2/assignment_loss‚0.3720351735573884 /loss/FRt1/detection_loss‚0.4696548283524617 .loss/FRt2/detection_loss‚0.462416619733817 trainer/global_step‚17899 _runtime‚4752.062251496 0loss/FRt1/assignment_loss‚0.4824876080707243 &loss/total_loss‚1.7865942297143915 epoch‚44à Å+ “í)‚(:ˆ î£ùÓêÎÆá gpu.0.gpu‚33.0 gpu.0.memory‚5, gpu.0.memoryAllocated‚7.0313262939453125) gpu.0.memoryAllocatedBytes‚ 6039863296 gpu.0.temp‚35.0 gpu.0.powerWatts‚125.88' gpu.0.enforcedPowerLimitWatts‚500.0 gpu.0.powerPercent‚25.176 gpu.0.smClock‚1410 gpu.0.memoryClock‚1593$ gpu.0.correctedMemoryErrors‚1955# gpu.0.uncorrectedMemoryErrors‚0ß“î)‚(:ˆ î£ùÓÈŸäâ$ disk.nvme0c0n1.in‚70047.08984375% disk.nvme0c0n1.out‚62257.27734375 disk.sda2.in‚0 disk.drbd2165.in‚0 disk.drbd2165.out‚0" disk.nvme0n1.in‚70047.08984375$ memory_percent‚68.61803531646729) proc.memory.availableMB‚ 5141.62109375# proc.memory.rssMB‚ 2354.24609375 proc.cpu.threads‚17) disk./.usagePercent‚32.72649785297745 disk.sda.in‚43542.96484375# disk.nvme0n1.out‚62257.27734375 disk.dm-0.in‚70047.08984375 disk.dm-0.out‚62257.27734375) proc.memory.percent‚14.36917781829834 cpu‚27.250770369827908 network.sent‚ 2.3327234e+07 disk.sda3.in‚43542.96484375 disk.sda.out‚ 87.828125 disk.sda1.in‚0 disk.sda1.out‚0 disk.sda2.out‚0 disk.dm-1.in‚43542.96484375 disk.dm-1.out‚ 87.828125 network.recv‚ 8.962859e+06$ disk./.usageGB‚487.6484680175781 disk.sda3.out‚ 87.828125—÷Ê2ï)- _wandbruntime‚4764  _runtime‚4764ÛÑHÉð)à $ _timestamp‚1.786663415519915e+09 _runtime‚ 4764.05186104 _step‚1476 ' lr-AdamW/pg1‚4.0163851693777897e-05 ' lr-AdamW/pg2‚4.0163851693777897e-05 trainer/global_step‚17949|ž¹Îñ)È $ _timestamp‚1.786663415519915e+09 _runtime‚ 4764.05186104 _step‚1476 ' lr-AdamW/pg1‚4.0163851693777897e-05 ' lr-AdamW/pg2‚4.0163851693777897e-05 trainer/global_step‚17949Ä ú¿,2ò)- _wandbruntime‚4764  _runtime‚4764/ü´Ä'ó)¾O VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚494036614649344 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17104896 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 4628521206 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 125829120 @7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚5264 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 4552982528 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚426882169274368 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 4588742277632 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚262 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚41945088 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚15058813 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚41945088 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚431470911552000 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 5494538240 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 trainer/global_step‚17949 M@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚59049984 BDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚494036555599360 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4289 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚322 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 122905600 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 125829120 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚322 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚489697798291456 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚ 126695391 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚209 UHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚68876288 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 831258624 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 5368709120 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 PCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚59049984 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚4287 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 4338774412800 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚4287 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚41348020 $ _timestamp‚1.786663415520567e+09 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚66445233 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 4338816357888 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 122905600 IBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚68 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 4329738767376 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 IBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚60 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚61478912 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 JDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚486804679000064 _runtime‚4764.051965377 C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚4289 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚ 126691102 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚60245869 PIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚66 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 122306526 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 5494538240 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚60250156 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17104896 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚494036614649344 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 4338774412800 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚60250156 F?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚60 F>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 G>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4876 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚494036555599360 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚66445233 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚56426418 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚15058747 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚431470842675712 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚489697798291456 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚223 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 5494538240 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚262 KBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚4876 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 4517085184 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚491134459115460 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚491134400728080 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 4654602240 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 4654602240 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚322 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚429 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚426882148630528ÃÿÉ'ô)ÃO N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 5494538240 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚262 KBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚4876 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 4517085184 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚491134459115460 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚491134400728080 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 4654602240 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 4654602240 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚322 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚429 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚426882148630528 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚494036614649344 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17104896 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 4628521206 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 125829120 @7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚5264 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 4552982528 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚426882169274368 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 4588742277632 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚262 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚41945088 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚15058813 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚41945088 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚431470911552000 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 5494538240 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 trainer/global_step‚17949 M@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚59049984 BDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚494036555599360 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4289 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚322 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 122905600 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 125829120 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚322 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚489697798291456 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚ 126695391 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚209 UHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚68876288 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 831258624 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 5368709120 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 PCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚59049984 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚4287 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 4338774412800 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚4287 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚41348020 $ _timestamp‚1.786663415520567e+09 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚66445233 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 4338816357888 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 122905600 IBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚68 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 4329738767376 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 IBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚60 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚61478912 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 JDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚486804679000064 _runtime‚4764.051965377 C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚4289 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚ 126691102 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚60245869 PIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚66 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 122306526 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 5494538240 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚60250156 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17104896 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚494036614649344 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 4338774412800 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚60250156 F?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚60 F>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 G>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4876 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚494036555599360 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚66445233 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚56426418 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚15058747 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚431470842675712 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚489697798291456 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚223Å ~n62õ)- _wandbruntime‚4764  _runtime‚4764§úK‚ö)ÉM OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚56429461 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚322 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚60249156 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 4588936258048 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚223 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚322 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚491160875481656 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚431494203231744 IBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚60 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚ 126702237 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚4293 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 125829120 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚494063168058880 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚489724120629248 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚60253449 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 4588986584576 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚41348068 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 4339005481472 SFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚70970368 G@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚60 EDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654602240 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 4339005481472 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚4293 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521206 H@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚665 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚ 126697942 PDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 4552982528 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 126702237 G@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚75 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚20643840 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚50326528 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 4517085184 trainer/global_step‚17949 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚66448786 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚489724120629248 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚ 126697942 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚56429386 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚61478912 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 5494538240 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚60249156 I@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚4876 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚15059651 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 122905600 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 122306526 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚262 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚4295 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚66448788 K>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59053056 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17104896 B9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚5264 B6DeviceStatsMonitor.on_trõFfMain_batch_end/segment.all.peak‚322 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚262 D=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚60 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚486830847754240 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 4654602240 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚494063168058880 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 _step‚1478 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚489724103524352 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 4552982528 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚66448786 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚41369808 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚15059578 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 831258624 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚491160817094228 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 4329969339988 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚429 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚431494132261376 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 125829120 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 5368709120 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚486830864793600 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚66448788 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 125829120 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 >5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5264 D8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 NADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚58387428 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 5368709120 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚60253449 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚322 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚426905216647168 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 5368709120çJJÌÔ&÷)ÎM WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚489724103524352 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 4552982528 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚66448786 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚41369808 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚15059578 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 831258624 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚491160817094228 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 4329969339988 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚429 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚431494132261376 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 125829120 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 5368709120 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚486830864793600 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚66448788 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 125829120 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 >5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5264 D8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 NADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚58387428 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 5368709120 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚60253449 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚322 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚426905216647168 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 5368709120 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚56429461 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚322 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚60249156 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 4588936258048 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚223 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚322 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚491160875481656 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚431494203231744 IBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚60 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚ 126702237 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚4293 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 125829120 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚494063168058880 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚489724120629248 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚60253449 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 4588986584576 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚41348068 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 4339005481472 SFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚70970368 G@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚60 EDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654602240 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 4339005481472 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚4293 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521206 H@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚665 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚ 126697942 PDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 4552982528 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 126702237 G@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚75 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚20643840 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚50326528 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 4517085184 trainer/global_step‚17949 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚66448786 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚489724120629248 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚ 126697942 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚56429386 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚61478912 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 5494538240 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚60249156 I@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚4876 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚15059651 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 122905600 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 122306526 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚262 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚4295 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚66448788 K>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59053056 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17104896 B9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚5264 B6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚322 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚262 D=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚60 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚486830847754240 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 4654602240 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚494063168058880 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 _step‚1478Æ ƒŠ"H2ø)- _wandbruntime‚4764  _runtime‚4764¸0'tù)î _step‚1479 /loss/FRt1/detection_loss‚0.4956522821497749 &loss/total_loss‚1.8370906092428254 trainer/global_step‚17949 _runtime‚4764.215968388 0loss/FRt1/assignment_loss‚0.4779365235750689 1loss/FRt2/assignment_loss‚0.38152788794820547 /loss/FRt2/detection_loss‚0.4819739155697762 epoch‚44 # _timestamp‚1.78666341568611e+09•xêWyú)ó _runtime‚4764.215968388 0loss/FRt1/assignment_loss‚0.4779365235750689 1loss/FRt2/assignment_loss‚0.38152788794820547 /loss/FRt2/detection_loss‚0.4819739155697762 epoch‚44 # _timestamp‚1.78666341568611e+09 _step‚1479 /loss/FRt1/detection_loss‚0.4956522821497749 &loss/total_loss‚1.8370906092428254 trainer/global_step‚17949Ç Ãü•¡û)‚(:– ý£ùÓǼƒá gpu.0.gpu‚64.0 gpu.0.memory‚12, gpu.0.memoryAllocated‚7.0313262939453125) gpu.0.memoryAllocatedBytes‚ 6039863296 gpu.0.temp‚36.0 gpu.0.powerWatts‚125.439' gpu.0.enforcedPowerLimitWatts‚500.0) gpu.0.powerPercent‚25.087799999999998 gpu.0.smClock‚1410 gpu.0.memoryClock‚1593$ gpu.0.correctedMemoryErrors‚1955# gpu.0.uncorrectedMemoryErrors‚0àØP[›ü)‚(: ý£ùÓÆ•èâ$ memory_percent‚68.61824989318848) disk./.usagePercent‚32.72649042875896$ disk.nvme0c0n1.in‚70047.08984375 disk.sda.out‚ 88.28515625 disk.sda2.out‚0) proc.memory.percent‚14.36917781829834 proc.cpu.threads‚17 network.recv‚ 8.993597e+06 disk.sda3.in‚43542.96484375& disk.nvme0c0n1.out‚62264.978515625 disk.sda.in‚43542.96484375 disk.dm-1.out‚ 88.28515625 disk.drbd2165.in‚0( proc.memory.availableMB‚ 5141.5859375# proc.memory.rssMB‚ 2354.24609375 network.sent‚ 2.3400771e+07$ disk./.usageGB‚487.6483573913574 disk.sda1.in‚0 disk.sda2.in‚0! disk.dm-0.out‚62264.978515625 cpu‚27.250559594623827 disk.sda3.out‚ 88.28515625 disk.sda1.out‚0" disk.nvme0n1.in‚70047.08984375$ disk.nvme0n1.out‚62264.978515625 disk.dm-1.in‚43542.96484375 disk.drbd2165.out‚0 disk.dm-0.in‚70047.08984375`¶ˆ52ý)- _wandbruntime‚4776  _runtime‚4776çP®Éþ)à % _timestamp‚1.7866634278150833e+09 _runtime‚4776.345094441 _step‚1480 & lr-AdamW/pg1‚3.824607930606844e-05 & lr-AdamW/pg2‚3.824607930606844e-05 trainer/global_step‚17999=lÿtÎÿ)È _runtime‚4776.345094441 _step‚1480 & lr-AdamW/pg1‚3.824607930606844e-05 & lr-AdamW/pg2‚3.824607930606844e-05 trainer/global_step‚17999 % _timestamp‚1.7866634278150833e+09È wnËp2€*- _wandbruntime‚4776  _runtime‚4776ÊÈ‘Ä'*¾O PCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚58387380 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚492455310602868 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 5494538240 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚488113968676864 PCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚59049984 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚60415258 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚56577689 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 125829120 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17104896 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚41477853 C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚4289 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚491013898076160 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 4350403123712 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚432635504124928 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 4600986854400 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚41477855 M@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚59049984 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 4350361178624 PIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚66 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 4654602240 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚41945088 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 5368709120 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚56577757 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 F?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚60 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚322 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚262 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚66622885 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚322 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 4350361178624 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 4552982528 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚4287 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚41348020 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 5494538240 D;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚5264 KBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚4876 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 4341300577984 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚4287 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17104896 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚491013898076160 JDeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4289 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 4601030892544 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 4628521206 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 5494538240 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 125829120 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚ 127033854 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚495364318304768 UHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚64681984 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚262 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚495364259254784 JBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚665 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚66622883 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 4552982528 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚209 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚492455252215488 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 5368709120 JDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4876 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 4654602240 IBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚60 F>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚15099902 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚44038144 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 4517085184 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚428034537914368 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚491013915181056 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚41945088 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 4341341926004 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 122306526 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚223 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚432635568806912GlàAj‚*ÃO G>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4876 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 4654602240 IBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚60 F>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚15099902 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚44038144 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 4517085184 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚428034537914368 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚491013915181056 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚41945088 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 4341341926004 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 122306526 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚223 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚432635568806912 PCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚58387380 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚492455310602868 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 5494538240 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚488113968676864 PCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚59049984 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚60415258 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚56577689 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 125829120 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17104896 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚41477853 C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚4289 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚491013898076160 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 4350403123712 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚432635504124928 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 4600986854400 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚41477855 M@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚59049984 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 4350361178624 PIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚66 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 4654602240 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚41945088 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 5368709120 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚56577757 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 F?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚60 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚322 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚262 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚66622885 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚322 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 4350361178624 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 4552982528 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚4287 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 WJDeviceStatsMonitor.on_train_batch_start/requëâžR_ested_bytes.small_pool.current‚41348020 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 5494538240 D;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚5264 KBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚4876 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 4341300577984 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚4287 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17104896 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚491013898076160 JDeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4289 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 4601030892544 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 4628521206 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 5494538240 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 125829120 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚ 127033854 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚495364318304768 UHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚64681984 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚262 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚495364259254784 JBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚665 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚66622883 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 4552982528 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚209 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚492455252215488 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 5368709120 J5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5264 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 4350592247296 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 K>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59053056 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 4350634195456 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 4350592247296 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521206 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚41948160 B9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚5264 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚60418551 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚491040237518848 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚432658795807744 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 P6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚322 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚322 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654602240 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚41948160 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚4293 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 4654602240 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 4552982528 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚41480058 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚262 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚ 127044989 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚41480060 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚15100729 D=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚60 DDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 127044989 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚56580716 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 4601231164416 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 122306526 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 5494538240 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 122905600 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚428057564643328 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚488140154470400 SFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚62581760 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚20643840 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 125829120 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚262 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚428057585287168 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚322 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚429 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚432658858389504 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 4341531150596 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 5494538240 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚322 _runtime‚4776.514103421 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚ 127040694 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚495390871714304 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚491040220413952 I@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚4876 G@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚73 BDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚66626436 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚66626436 NGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚71 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚60414258 NADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚59053056 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17104896 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚223 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚488140137431040 _step‚1482OÝÞëÓ&…*ÍM P6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚322 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚322 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654602240 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚41948160 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚4293 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 4654602240 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 4552982528 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚41480058 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚262 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚ 127044989 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚41480060 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚15100729 D=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚60 DDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 127044989 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚56580716 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 4601231164416 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 122306526 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 5494538240 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 122905600 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚428057564643328 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚488140154470400 SFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚62581760 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚20643840 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 125829120 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚262 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚428057585287168 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚322 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚429 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚432658858389504 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 4341531150596 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 5494538240 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚322 _runtime‚4776.514103421 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚ 127040694 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚495390871714304 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚491040220413952 I@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚4876 G@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚73 BDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚66626436 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚66626436 NGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚71 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚60414258 NADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚59053056 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17104896 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚223 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚488140137431040 _step‚1482 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 NADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚58387428 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 5494538240 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 5368709120 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 5368709120 $ _timestamp‚1.786663427983803e+09 E5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5264 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 4350592247296 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 K>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59053056 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 4350634195456 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 4350592247296 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521206 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚41948160 B9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚5264 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚60418551 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚491040237518848 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚432658795807744 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0Ê Ÿ€ùå2†*- _wandbruntime‚4776  _runtime‚4776ò[Á9v‡*ð 0loss/FRt2/detection_loss‚0.44852112412574996 &loss/total_loss‚1.8382501202717343 epoch‚44 trainer/global_step‚17999 % _timestamp‚1.7866634279848714e+09 _step‚1483 0loss/FRt1/assignment_loss‚0.5050048216198922 /loss/FRt1/detection_loss‚0.4986466527864733 _runtime‚4776.514717161 0loss/FRt2/assignment_loss‚0.3860775217396186xÎýn{ˆ*õ _runtime‚4776.514717161 0loss/FRt2/assignment_loss‚0.3860775217396186 0loss/FRt2/detection_loss‚0.44852112412574996 &loss/total_loss‚1.8382501202717343 epoch‚44 trainer/global_step‚17999 % _timestamp‚1.7866634279848714e+09 _step‚1483 0loss/FRt1/assignment_loss‚0.5050048216198922 /loss/FRt1/detection_loss‚0.4986466527864733Ë «ŒÙê2‰*- _wandbruntime‚4781  _runtime‚4781[CiÜŠ*Ö *jet/accuracy_2_of_2‚0.2116740494966507 6validation_average_jet_accuracy‚0.3150864243507385 epoch‚44 2Purity/2FRt/event_proportion‚0.040062315762043 *jet/accuracy_1_of_1‚0.3101128935813904 *jet/accuracy_1_of_2‚0.5733587145805359 /particle/accuracy_2_of_2‚0.6782214045524597 (particle/accuracy‚0.6398460865020752 +particle/sensitivity‚0.6289390921592712 *loss/val_total_loss‚2.5301348635786094 particle/accuracy_1_of_1‚1 Purity/0FRt/event_purity‚1 /Purity/2FRt/event_purity‚0.4290417730808258 *particle/specificity‚0.645828902721405 'particle/f_score‚0.5534900426864624 *validation_accuracy‚0.2116740494966507 _runtime‚4781.519968875 trainer/global_step‚17999 % _timestamp‚1.7866634329899316e+09 /Purity/*FRt/event_purity‚0.4660990536212921 .Purity/*FRt/FRt_purity‚0.47935566306114197 3Purity/0FRt/event_proportion‚0.3301943838596344 /Purity/1FRt/event_purity‚0.4683355689048767 /particle/accuracy_1_of_2‚0.9213564991950989 _step‚1484 2Purity/*FRt/event_proportion‚0.669805645942688 -Purity/2FRt/FRt_purity‚0.5644828081130981 3Purity/1FRt/event_proportion‚0.6297432780265808 -Purity/1FRt/FRt_purity‚0.4683355689048767[mæÉá‹*Û *particle/specificity‚0.645828902721405 'particle/f_score‚0.5534900426864624 *validation_accuracy‚0.2116740494966507 _runtime‚4781.519968875 trainer/global_step‚17999 % _timestamp‚1.7866634329899316e+09 /Purity/*FRt/event_purity‚0.4660990536212921 .Purity/*FRt/FRt_purity‚0.47935566306114197 3Purity/0FRt/event_proportion‚0.3301943838596344 /Purity/1FRt/event_purity‚0.4683355689048767 /particle/accuracy_1_of_2‚0.9213564991950989 _step‚1484 2Purity/*FRt/event_proportion‚0.669805645942688 -Purity/2FRt/FRt_purity‚0.5644828081130981 3Purity/1FRt/event_proportion‚0.6297432780265808 -Purity/1FRt/FRt_purity‚0.4683355689048767 *jet/accuracy_2_of_2‚0.2116740494966507 6validation_average_jet_accuracy‚0.3150864243507385 epoch‚44 2Purity/2FRt/event_proportion‚0.040062315762043 *jet/accuracy_1_of_1‚0.3101128935813904 *jet/accuracy_1_of_2‚0.5733587145805359 /particle/accuracy_2_of_2‚0.6782214045524597 (particle/accuracy‚0.6398460865020752 +particle/sensitivity‚0.6289390921592712 *loss/val_total_loss‚2.5301348635786094 particle/accuracy_1_of_1‚1 Purity/0FRt/event_purity‚1 /Purity/2FRt/event_purity‚0.4290417730808258Ì ?·±¡Œ*‚(:– Œ¤ùÓª‡á gpu.0.gpu‚46.0 gpu.0.memory‚10, gpu.0.memoryAllocated‚7.0313262939453125) gpu.0.memoryAllocatedBytes‚ 6039863296 gpu.0.temp‚33.0 gpu.0.powerWatts‚119.415' gpu.0.enforcedPowerLimitWatts‚500.0) gpu.0.powerPercent‚23.883000000000003 gpu.0.smClock‚1410 gpu.0.memoryClock‚1593$ gpu.0.correctedMemoryErrors‚1955# gpu.0.uncorrectedMemoryErrors‚0pý‘*‚(:† Œ¤ùÓ¤¡›ã disk.sda1.in‚0 disk.sda2.out‚0& disk.nvme0n1.out‚62270.32275390625 disk.dm-0.in‚ 70058.28125! disk.nvme0c0n1.in‚ 70058.28125( disk.nvme0c0n1.out‚62270.32275390625" proc.memory.rssMB‚ 2299.5546875 cpu‚27.255271011435145 network.sent‚ 2.3473588e+07' disk./.usagePercent‚32.726501949098 disk.sda1.out‚0# disk.dm-0.out‚62270.32275390625 disk.sda3.out‚ 88.51171875 disk.sda.in‚ 43562.7734375 disk.sda.out‚ 88.51171875 network.recv‚ 9.023656e+06$ disk./.usageGB‚487.6485290527344 disk.dm-1.in‚ 43562.7734375 disk.drbd2165.in‚0 disk.drbd2165.out‚0$ memory_percent‚68.28937530517578* proc.memory.percent‚14.035367965698242 proc.cpu.threads‚17 disk.sda2.in‚0 disk.nvme0n1.in‚ 70058.28125 disk.dm-1.out‚ 88.51171875 disk.sda3.in‚ 43562.7734375& proc.memory.availableMB‚ 5195.46875/è²R2Ž*- _wandbruntime‚4793  _runtime‚4793ÕÉ*à & lr-AdamW/pg1‚3.637402154266425e-05 & lr-AdamW/pg2‚3.637402154266425e-05 trainer/global_step‚18049 % _timestamp‚1.7866634451151326e+09 _runtime‚4793.645144611 _step‚1485ÍK¬§Î*È trainer/global_step‚18049 % _timestamp‚1.7866634451151326e+09 _runtime‚4793.645144611 _step‚1485 & lr-AdamW/pg1‚3.637402154266425e-05 & lr-AdamW/pg2‚3.637402154266425e-05Í ·˜€q2‘*- _wandbruntime‚4793  _runtime‚4793˜5V\ ’*¾O NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚4287 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 G>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4876 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚497087026898944 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚492720952180736 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 125829120 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚322 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 5368709120 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚66861139 M@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚59049984 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚434103522126336 $ _timestamp‚1.786663445116037e+09 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 831258624 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17104896 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚497087026898944 JBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚665 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 4517085184 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚492720969285632 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚41945088 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 125829120 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚209 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚20643840 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚322 YEDeviceStatsMonitße­Ç¬or.on_train_batch_start/requested_bytes.all.allocated‚494161127554217 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 KDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚60 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 4366015668224 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 125829120 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚497086967848960 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚ 127459399 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 122306526 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 4628521206 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 4356928360693 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 5494538240 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 F>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚60602547 UHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚64681984 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 122905600 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 122905600 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 4617365907968 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 4356969708713 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚322 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚41945088 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 KBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚4876 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚497086967848960 IBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚60 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 845178880 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 5368709120 trainer/global_step‚18049 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚56778634 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚262 _step‚1486 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 5368709120 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚ 127463688 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4289 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 4552982528 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 4366015668224 IBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚60 JDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4876 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚497087026898944 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚492720952180736 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 125829120 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚322 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 5368709120 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚66861139 M@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚59049984 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚434103522126336 $ _timestamp‚1.786663445116037e+09 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 831258624 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17104896 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚497087026898944 JBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚665 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 4517085184 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚492720969285632 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚41945088 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 125829120 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚209 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚20643840 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚322 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚494161127554217 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 KDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚60 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 4366015668224 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 125829120 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚497086967848960 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚ 127459399 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 122306526 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 4628521206 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 4356928360693 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 5494538240 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 F>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚60602547 UHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚64681984 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 122905600 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 122905600 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 4617365907968 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 4356969708713 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚322 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚41945088 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 KBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚4876 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚497086967848960 IBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚60 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 845178880 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 5368709120 trainer/global_step‚18049 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚56778634 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚262 _step‚1486 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 5368709120 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚ 127463688 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4289 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 4552982528 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 4366015668224 IBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚60 JDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59053056 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚489830343639040 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 4357200281373 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚60605840 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚492747291623424 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚66864692 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 831258624 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521206 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 4654602240 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 D=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚60 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 5494538240 % _timestamp‚1.7866634452794297e+09 >5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5264 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17104896 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 4552982528 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 127470534 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚ 127466239 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 845178880 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚46132224 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚41348068 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 NGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚67 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚61478912 _runtime‚4793.809769002 H@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚665 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚56781671 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 125829120 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 125829120 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 125829120 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 5368709120 _step‚1487 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚429509203591168 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚494187543920413 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚322 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚262 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚4295 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 4366246736896 B9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚5264 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚66864694 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚497113580308480 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚322 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 5494538240 EDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚56781602 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚223 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚15150680 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 BDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654602240 E6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚322 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0Â?„Eg–*ÎM J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 4654602240 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 D=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚60 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 5494538240 % _timestamp‚1.7866634452794297e+09 >5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5264 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17104896 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 4552982528 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 127470534 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚ 127466239 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 845178880 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚46132224 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚41348068 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 NGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚67 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚61478912 _runtime‚4793.809769002 H@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚665 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚56781671 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 125829120 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 125829120 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 125829120 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 5368709120 _step‚1487 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚429509203591168 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚494187543920413 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚322 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚262 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚4295 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 4366246736896 B9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚5264 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚66864694 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚497113580308480 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚322 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 5494538240 EDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚56781602 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚223 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚15150680 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 BDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654602240 E6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚322 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 IBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚60 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 5368709120 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚497113580308480 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚41630924 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚429509224235008 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚492747291623424 G@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚60 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚ 127470534 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚66864692 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚60601547 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 5368709120 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚4293 K>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59053056 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚489830343639040 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 4357200281373 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚60605840 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚492747291623424 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚66864692 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 831258624 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521206Ï _v²ä2—*- _wandbruntime‚4793  _runtime‚4793®íu˜*ï 0loss/FRt1/assignment_loss‚0.5057777034939693 0loss/FRt2/assignment_loss‚0.3707499243283765 epoch‚45 % _timestamp‚1.7866634452804806e+09 _runtime‚4793.810341885 /loss/FRt1/detection_loss‚0.4640774629055263 /loss/FRt2/detection_loss‚0.4753249582402365 &loss/total_loss‚1.8159300489681087 trainer/global_step‚18049 _step‚1488¤Áþz™*ô &loss/total_loss‚1.8159300489681087 trainer/global_step‚18049 _step‚1488 0loss/FRt1/assignment_loss‚0.5057777034939693 0loss/FRt2/assignment_loss‚0.3707499243283765 epoch‚45 % _timestamp‚1.7866634452804806e+09 _runtime‚4793.810341885 /loss/FRt1/detection_loss‚0.4640774629055263 /loss/FRt2/detection_loss‚0.4753249582402365Ð ãù>”š*‚(:‰ ›¤ùÓ£§öà gpu.0.gpu‚64.0 gpu.0.memory‚12, gpu.0.memoryAllocated‚7.0313262939453125) gpu.0.memoryAllocatedBytes‚ 6039863296 gpu.0.temp‚36.0 gpu.0.powerWatts‚175.33' gpu.0.enforcedPowerLimitWatts‚500.0 gpu.0.powerPercent‚35.066 gpu.0.smClock‚1410 gpu.0.memoryClock‚1593$ gpu.0.correctedMemoryErrors‚1955# gpu.0.uncorrectedMemoryErrors‚0ž7 ÿ‰›*‚(:þ ›¤ùÓž–õâ( disk.nvme0c0n1.out‚62277.09326171875 disk.sda1.out‚0 cpu‚27.254999750570292 disk.nvme0n1.in‚ 70133.125& disk.nvme0n1.out‚62277.09326171875 disk.dm-1.in‚ 44704.5859375# disk.dm-0.out‚62277.09326171875 disk.sda.in‚ 44704.5859375 disk.sda.out‚ 88.640625 disk.sda2.in‚0$ memory_percent‚68.29063892364502 network.recv‚ 9.043546e+06$ disk./.usageGB‚487.6485061645508 disk.drbd2165.in‚0 disk.sda2.out‚0) proc.memory.availableMB‚ 5195.26171875" proc.memory.rssMB‚ 2299.5546875* proc.memory.percent‚14.035367965698242 proc.cpu.threads‚17( disk./.usagePercent‚32.7265004130528 disk.dm-0.in‚ 70133.125 disk.sda3.in‚ 44704.5859375 disk.nvme0c0n1.in‚ 70133.125 disk.sda1.in‚0 network.sent‚ 2.3538413e+07 disk.dm-1.out‚ 88.640625 disk.drbd2165.out‚0 disk.sda3.out‚ 88.640625 : Þ2œ*- _wandbruntime‚4805  _runtime‚4805|ËóÇ*Á % lr-AdamW/pg1‚3.45477986423377e-05 % lr-AdamW/pg2‚3.45477986423377e-05 trainer/global_step‚18099 % _timestamp‚1.7866634572147622e+09 _runtime‚4805.744877655 _step‚1489KóŠÌž*Æ _runtime‚4805.744877655 _step‚1489 % lr-AdamW/pg1‚3.45477986423377e-05 % lr-AdamW/pg2‚3.45477986423377e-05 trainer/global_step‚18099 % _timestamp‚1.7866634572147622e+09Ñ }M9”2Ÿ*- _wandbruntime‚4805  _runtime‚4805¶c(Å' *¿O aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚430638524858368 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 5494538240 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 5368709120 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚495481920654245 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚262 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚4287 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚60767649 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚15190462 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 4629650098176 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 5494538240 KDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚60 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚ 127806440 IBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚67 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 831258624 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 4377602434048 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 4552982528 JDeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4289 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 125829120 IBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚60 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚56929496 F?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚60 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚262 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 4377644379136 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚67038789 PIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚65 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚430638545502208 BDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4876 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚56929429 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚61478912 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 5494538240 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 4552982528 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 4517085184 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚322 _step‚1490 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚67038791 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚223 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚209 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚491113447522304 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚322 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17104896 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚ 127802151 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 4654602240 trainer/global_step‚18099 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 JDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚498414671504384 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚41945088 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 125829120 _runtime‚4805.746046304 F>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚15190527 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚322 JBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚665‚¶ëÊ'¡*ÄO UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚495481920654245 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚262 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚4287 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚60767649 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚15190462 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 4629650098176 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 5494538240 KDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚60 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚ 127806440 IBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚67 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 831258624 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 4377602434048 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 4552982528 JDeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4289 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 125829120 IBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚60 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚56929496 F?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚60 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚262 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 4377644379136 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚67038789 PIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚65 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚430638545502208 BDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4876 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚56929429 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚61478912 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 5494538240 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 4552982528 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 4517085184 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚322 _step‚1490 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚67038791 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚223 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚209 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚491113447522304 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚322 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17104896 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚ 127802151 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 4654602240 trainer/global_step‚18099 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 JDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚498414671504384 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚41945088 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 125829120 _runtime‚4805.746046304 F>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚15190527 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚322 JBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚665 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚430638524858368 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 5494538240 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 5368709120Ò 9ÛĘ2¢*- _wandbruntime‚4805  _runtime‚4805.0Š_;£*ÈM E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚67042342 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚15191293 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚262 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚60766649 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 4654602240 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚494063391408128 NADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚59053056 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚ 127808991 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚322 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚67042344 G@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚56 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚498441224910848 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚429 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚20643840 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 IBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚60 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17104896 EDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654602240 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 4552982528 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 122306526 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 4629938476032 D=DeviceStatsMonitor.on_train_€TÔ„“batch_end/segment.small_pool.peak‚60 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚67042342 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚56932411 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 G@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚60 DDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 127813286 I@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚4876 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚262 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚60770942 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚67042344 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚491139633315840 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 4368720743913 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 125829120 K>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59053056 B9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚5264 SFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚60484608 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚495508395407821 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚41741174 >5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5264 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 4552982528 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚41948160 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚322 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚495508337020393 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521206 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚498441283963904 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚39840768 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 125829120 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 4377833502720 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 122905600 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚435291470866432 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 831258624 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 B6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚322 NGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚54 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚15191239 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 E6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚322 NGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚54 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚15191239 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 E8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 IBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚60 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17104896 EDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654602240 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 4552982528 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 122306526 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 4629938476032 D=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚60 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚67042342 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚56932411 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 G@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚60 DDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 127813286 I@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚4876 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚262 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚60770942 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚67042344 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚491139633315840 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 4368720743913 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 125829120 K>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59053056 B9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚5264 SFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚60484608 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚495508395407821 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚41741174 >5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5264 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 4552982528 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚41948160 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚322 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚495508337020393 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521206 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚498441283963904 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚39840768 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 125829120 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 4377833502720 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 122905600 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚435291470866432 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 831258624 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 BDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚499742375159808 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚41945088 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 122905600 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 4641989686272 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 5494538240 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚322 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17104896 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚499742375159808 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 4389231144960 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚495353185959936 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚57080333 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚41849217 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚ 128149192 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 122306526 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚15231188 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚495353185959936 JDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4876 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚495353203064832 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17104896 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚429 UHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚73070592 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚61478912 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚20643840 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚496802830529033 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 5368709120 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚322 KDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚60 _step‚1494 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4289 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚67216439 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚431790893498368 BDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 125829120 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚322 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 4654602240 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 4389189199872 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚41849219 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚436432830757888 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚431790914142208 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 4654602240 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 4552982528 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 831258624 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚492422720159744 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚15231116 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 5494538240 JDeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4289 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚67216439 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚431790893498368 BDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 125829120 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚322 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 4654602240 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 4389189199872 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚41849219 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚436432830757888 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚431790914142208 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 4654602240 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 4552982528 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 831258624 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚492422720159744 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚15231116 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 5494538240 JDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚499742375159808 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚41945088 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 122905600 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 4641989686272 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 5494538240 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚322 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17104896 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚499742375159808 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 4389231144960 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚495353185959936 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚57080333 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚41849217 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚ 128149192 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 122306526 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚15231188 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚495353185959936 JDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4876 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚495353203064832 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17104896Ö ì—›‹2°*- _wandbruntime‚4818  _runtime‚4818@vªÑÎ&±*ÈM SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 5368709120 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 4389462216704 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 4642180520960 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 5494538240 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 831258624 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚223 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 NADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚59053056 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17104896 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚67219992 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 125829120 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚262 _runtime‚4818.077630354 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚60936044 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521206 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚60931751 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚495379508297728 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚209 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚20643840 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚262 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 4389462216704 NGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚59 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚ 128151743 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚436456194459648 SFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚73067520 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚496829188507801 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚4293 D5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5264 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚67219992 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 4389420268544 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚322 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 4517085184 trainer/global_step‚18149 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚4295 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚41948160 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚499768987619328 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚41851422 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚492448905953280 G@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚61 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 122905600 B9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚5264 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚499768928566272 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚15231900 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚41348068 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 5368709120 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 125829120 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚60936044 K>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59053056 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 $ _timestamp‚1.786663469547366e+09 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚ 128156038 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 NADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚58387428 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚322 D=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚60 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚ 128151743 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17104896 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 4380282554521 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚322 EDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654602240 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚495379525402624 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚495379508297728 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 128156038 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚67219994 _step‚1495 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚41948160 EDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 4552982528 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚41851424 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚436456121392128 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚61478912 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚322 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 5368709120 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 4389420268544 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚429 BDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚57083322 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚431813940871168 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚262 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚499768987619328 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚57083383 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 4380323902589 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚67219994 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚4293 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚60931751 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚15231959 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 4552982528 I@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚4876 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 845178880 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 5494538240cD$Ó&²*ÍM H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚ 128151743 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚436456194459648 SFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚73067520 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚496829188507801 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚4293 D5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5264 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚67219992 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 4389420268544 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚322 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 4517085184 trainer/global_step‚18149 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚4295 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚41948160 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚499768987619328 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚41851422 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚492448905953280 G@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚61 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 122905600 B9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚5264 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚499768928566272 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚15231900 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚41348068 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 5368709120 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 125829120 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚60936044 K>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59053056 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 $ _timestamp‚1.786663469547366e+09 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚ 128156038 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 NADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚58387428 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚322 D=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚60 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚ 128151743 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17104896 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 4380282554521 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚322 EDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654602240 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚495379525402624 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚495379508297728 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 128156038 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚67219994 _step‚1495 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚41948160 EDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 4552982528 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚41851424 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚436456121392128 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚61478912 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚322 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 5368709120 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 4389420268544 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚429 BDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚57083322 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚431813940871168 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚262 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚499768987619328 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚57083383 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 4380323902589 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚67219994 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚4293 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚60931751 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚15231959 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 4552982528 I@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚4876 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 845178880 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 5494538240 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 5368709120 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 4389462216704 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 4642180520960 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 5494538240 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 831258624 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚223 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 NADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚59053056 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17104896 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚67219992 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 125829120 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚262 _runtime‚4818.077630354 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚60936044 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521206 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚60931751 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚495379508297728 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚209 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚20643840 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚262 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 4389462216704 NGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚59× ˜àÁ2³*- _wandbruntime‚4818  _runtime‚4818€ó/u´*ï /loss/FRt2/detection_loss‚0.4700513335737159 &loss/total_loss‚1.8151843017364833 epoch‚45 % _timestamp‚1.7866634695483117e+09 _runtime‚4818.078100583 0loss/FRt1/assignment_loss‚0.4844201825462892 0loss/FRt2/assignment_loss‚0.3924593491066004 /loss/FRt1/detection_loss‚0.4682534365098777 trainer/global_step‚18149 _step‚1496É„;DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚501070078815232 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚322 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 4654602240 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚61097853 UHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚70973440 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 4391655140537 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 5494538240 M@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚59049984 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 4400817910784 JDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4876 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚67394091 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚437597473569280 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 4628521206 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚223 PCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚58387380 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚262 D;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚5264 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚322 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 845178880 QBDeviceStatsMonitor.on_train_batch_sta¦"ñ\Ûrt/reserved_bytes.all.current‚ 5494538240 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚61093566 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚432943282782208 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚498123623629061 _runtime‚4830.353864779 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚57231087 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17104896 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 4400775965696 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 125829120 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚67394089 KBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚4876 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚15271620 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚41348020 C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚4289 @7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚5264 F>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 4552982528 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚322 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 125829120 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚501070137865216 PIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚69 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 4654211430912 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚61097853 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚493732009836544 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚496669319954432 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 5368709120 F?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚60 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚262 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 4400775965696 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚41945088 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚67394091 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚41959469 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 4654261760512 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 PCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚59049984 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 4400817910784 JDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚496669319954432 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚496669302849536 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚41945088 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚493732026875904 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 125829120 trainer/global_step‚18199 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚ 128491944 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚501070078815232 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚432943262138368 IBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚60 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚ 128487655 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 122905600 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚262 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 4654602240 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 4552982528 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4289 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚41959467 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚209 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚20643840 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 5368709120vÚ¿4É'½*ÃO VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚501070137865216 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 5368709120 _step‚1498 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 122905600 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚429 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 4391613792517 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚4287 BDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4876 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚67394091 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚437597473569280 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 4628521206 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚223 PCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚58387380 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚262 D;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚5264 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚322 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 845178880 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 5494538240 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚61093566 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚432943282782208 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚498123623629061 _runtime‚4830.353864779 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚57231087 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17104896 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 4400775965696 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 125829120 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚67394089 KBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚4876 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚15271620 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚41348020 C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚4289 @7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚5264 F>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 4552982528 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚322 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 125829120 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚501070137865216 PIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚69 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 4654211430912 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚61097853 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚493732009836544 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚496669319954432 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 5368709120 F?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚60 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚262 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 4400775965696 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚41945088 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚67394091 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚41959469 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 4654261760512 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 PCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚59049984 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 4400817910784 JDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚496669319954432 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚496669302849536 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚41945088 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚493732026875904 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 125829120 trainer/global_step‚18199 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚ 128491944 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚501070078815232 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚432943262138368 IBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚60 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚ 128487655 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 122905600 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚262 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 4654602240 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 4552982528 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4289 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚41959467 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚209 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚20643840 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 5368709120 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚501070078815232 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚322 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 4654602240 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚61097853 UHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚70973440 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 4391655140537 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 5494538240 M@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚59049984 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 4400817910784 J6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚322 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 EDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 BDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 125829120 IBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚60 D=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚60 $ _timestamp‚1.786663481981153e+09 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 4401007034368 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚262 K>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59053056 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚501096632221696 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚496695625187328 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚57234102 NGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚67 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚498150098382637 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚67397642 PDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17104896 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚501096691274752 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 4401007034368 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 125829120 _step‚1499 NADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚59053056 B9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚5264 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚67397642 I@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚4876 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚20643840 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚48229376 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 122306526 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 4401048982528 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚432966330155008 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚432966309511168 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 5494538240 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 5368709120 trainer/global_step‚18199 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚496695625187328 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚61096853 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚15272430 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚437620835894784 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚496695642292224 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 4552982528 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚223 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚61478912 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521206 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚41348068 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 122905600 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚61096853 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 4654602240 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚41948160 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚322 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚41948160 G@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚69 >5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5264 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 831258624 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 845178880 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚322 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚322 G@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚60 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚41961674 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚496695642292224 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17104896 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚67397644 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 5494538240 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 128498790 EDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654602240 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 4552982528 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 125829120 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚4293 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚501096691274752 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 H@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚665 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚262 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚429 SFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚68873216 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 4654457510400»Ýæ9À*ÍM EDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 BDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 125829120 IBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚60 D=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚60 $ _timestamp‚1.786663481981153e+09 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 4401007034368 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚262 K>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59053056 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚501096632221696 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚496695625187328 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚57234102 NGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚67 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚498150098382637 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚67397642 PDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17104896 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚501096691274752 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 4401007034368 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 125829120 _step‚1499 NADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚59053056 B9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚5264 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚67397642 I@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚4876 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚20643840 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚48229376 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 122306526 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 4401048982528 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚432966330155008 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚432966309511168 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 5494538240 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 5368709120 trainer/global_step‚18199 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚496695625187328 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚61096853 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚15272430 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚437620835894784 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚496695642292224 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 4552982528 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚223 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚61478912 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521206 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚41348068 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 122905600 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚61096853 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 4654602240 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚41948160 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚322 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚41948160 G@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚69 >5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5264 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 831258624 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 845178880 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚322 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚322 G@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚60 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚41961674 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚496695642292224 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.currÙW@Sš ent‚17104896 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚67397644 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 5494538240 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 128498790 EDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654602240 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 4552982528 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 125829120 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚4293 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚501096691274752 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 H@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚665 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚262 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚429 SFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚68873216 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 4654457510400 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚4295 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚41961672 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚209 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚493758195630080 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 4391885713197 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚322 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0Û qœì<2Á*- _wandbruntime‚4830  _runtime‚4830kŸ@yÂ*ó trainer/global_step‚18199 _runtime‚4830.511962739 _step‚1500 0loss/FRt1/detection_loss‚0.46867373187002925 epoch‚45 % _timestamp‚1.7866634819821532e+09 1loss/FRt1/assignment_loss‚0.42250130106241296 1loss/FRt2/assignment_loss‚0.33652279303008664 0loss/FRt2/detection_loss‚0.49093713859505195 &loss/total_loss‚1.7186349645575807¥Ø£I~Ã*ø 1loss/FRt1/assignment_loss‚0.42250130106241296 1loss/FRt2/assignment_loss‚0.33652279303008664 0loss/FRt2/detection_loss‚0.49093713859505195 &loss/total_loss‚1.7186349645575807 trainer/global_step‚18199 _runtime‚4830.511962739 _step‚1500 0loss/FRt1/detection_loss‚0.46867373187002925 epoch‚45 % _timestamp‚1.7866634819821532e+09Ü Ò!á2Ä*- _wandbruntime‚4842  _runtime‚4842SHa£ËÅ*Å _step‚1501 ' lr-AdamW/pg1‚2.9345297304215397e-05 ' lr-AdamW/pg2‚2.9345297304215397e-05 trainer/global_step‚18249 % _timestamp‚1.7866634939136631e+09 _runtime‚4842.443796245—UuÐÆ*Ê ' lr-AdamW/pg1‚2.9345297304215397e-05 ' lr-AdamW/pg2‚2.9345297304215397e-05 trainer/global_step‚18249 % _timestamp‚1.7866634939136631e+09 _runtime‚4842.443796245 _step‚1501Ý ¦u¸«2Ç*- _wandbruntime‚4842  _runtime‚4842ô%_Ä'È*¾O ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 5368709120 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚429 JDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚41945088 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚67571741 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚209 trainer/global_step‚18249 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 4412362731520 PCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚59049984 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚223 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚20643840 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 122306526 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚497985436844032 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 845178880 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 4666536826368 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚4287 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 F?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚60 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 4412404676608 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚434095651422208 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 5368709120 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚322 G>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4876 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚502397841520640 IBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚66 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 4517085184 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚67571739 JBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚665 UHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚62584832 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚499444533503849 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 4552982528 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 125829120 M@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚59049984 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚502397782470656 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚41945088 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 4403216951145 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 4654602240 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 122905600 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚57382018 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 PCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚58387380 _runtime‚ 4842.44788412 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚42069717 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚497985419739136 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4289 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚438762125663744 _step‚1502 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚495041316552704 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚ 128830407 JDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚495041299513344 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚322 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚502397782470656 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17104896 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 125829120 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 122905600 D;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚5264 KBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚4876 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚438762188248576 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚322 KDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚60 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚502397841520640 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 5494538240 IBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚60 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚61258668 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚15312235 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 5494538240 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚67571741 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 @7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚5264 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚61262955 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 831258624 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚499444475116469 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚ 128834696 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚42069719‘5hªÉ'É*ÃO NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚4287 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 F?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚60 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 4412404676608 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚434095651422208 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 5368709120 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚322 G>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4876 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚502397841520640 IBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚66 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 4517085184 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚67571739 JBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚665 UHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚62584832 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚499444533503849 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 4552982528 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 125829120 M@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚59049984 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚502397782470656 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚41945088 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 4403216951145 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 4654602240 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 122905600 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚57382018 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 PCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚58387380 _runtime‚ 4842.44788412 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚42069717 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚497985419739136 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4289 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚438762125663744 _step‚1502 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚495041316552704 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚ 128830407 JDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚495041299513344 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚322 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚502397782470656 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17104896 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 125829120 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 122905600 D;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚5264 KBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚4876 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚438762188248576 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚322 KDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚60 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚502397841520640 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 5494538240 IBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚60 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚61258668 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚15312235 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 5494538240 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚67571741 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 @7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚5264 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚61262955 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 831258624 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚499444475116469 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚ 128834696 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚42069719 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 5368709120 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚429 JDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚41945088 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚67571741 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚209 trainer/global_step‚18249 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 4412362731520 PCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚59049984 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚223 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚20643840 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 122306526 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚497985436844032 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 845178880 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 4666536826368 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0Þ $ôÕ2Ê*- _wandbruntime‚4842  _runtime‚48425ß +Ë*ÉM UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 4412593800192 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚429 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚57385012 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 122905600 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚322 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚67575294 P8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚61261955 DDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 125829120 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚438785412627968 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚498011742076928 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚42071924 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚434118698795008 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚502424335877120 H@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚665 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚42071922 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚67575292 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚61266248 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚61261955 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 SFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚70970368 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚499470891482617 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 4403447523805 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚223 K>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59053056 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 5494538240 G@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚60 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚498011759181824 G@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚80 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 5368709120 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚322 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚67575292 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚41948160 >5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5264 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 NADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚59053056 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚ 128837247 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 D=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚60 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚61266248 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚502424394930176 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚15313088 BDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚495067502346240 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚262 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 4666784803328 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚498011759181824 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 845178880 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚499470949870045 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 4517085184 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚502424394930176 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚ 128841542 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 831258624 EDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚57384932 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚438785483598336 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚4295 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚15313010 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚498011742076928 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 4552982528 \JDeviceStatsMonitor.on_train_batch_Ï ¤end/allocated_bytes.small_pool.allocated‚ 4412635748352 B9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚5264 NADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚58387428 IBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚60 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚ 128837247 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17104896 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 128841542 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚67575294 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚50326528 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚61478912 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 5368709120 trainer/global_step‚18249 _runtime‚4842.609152559 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚4293 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚41948160 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚4293 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 5494538240 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 125829120 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚322 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654602240 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521206 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 122306526 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17104896 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚322 % _timestamp‚1.7866634940788565e+09 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 4552982528 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 NGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚78 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚262 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚262 QB®Ô&Ì*ÎM S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚502424335877120 H@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚665 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚42071922 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚67575292 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚61266248 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚61261955 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 SFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚70970368 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚499470891482617 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 4403447523805 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚223 K>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59053056 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 5494538240 G@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚60 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚498011759181824 G@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚80 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 5368709120 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚322 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚67575292 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚41948160 >5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5264 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 NADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚59053056 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚ 128837247 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 D=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚60 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚61266248 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚502424394930176 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚15313088 BDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚495067502346240 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚262 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 4666784803328 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚498011759181824 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 845178880 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚499470949870045 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 4517085184 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚502424394930176 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚ 128841542 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 831258624 EDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚57384932 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚438785483598336 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚4295 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚15313010 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚498011742076928 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 4552982528 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 4412635748352 B9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚5264 NADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚58387428 IBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚60 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚ 128837247 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17104896 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 128841542 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚67575294 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚50326528 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚61478912 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 5368709120 trainer/global_step‚18249 _runtime‚4842.609152559 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚4293 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚41948160 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚4293 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 5494538240 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 125829120 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚322 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654602240 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521206 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 122306526 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17104896 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚322 % _timestamp‚1.7866634940788565e+09 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 4552982528 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 NGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚78 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚262 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚262 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 4412593800192 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚429 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚57385012 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 122905600 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚322 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚67575294 P8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚61261955 DDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 125829120 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚438785412627968 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚498011742076928 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚42071924 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚434118698795008 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0ß ß@žÏ2Í*- _wandbruntime‚4842  _runtime‚4842}èÿ»uÎ*ï 0loss/FRt2/assignment_loss‚0.3679503691313353 /loss/FRt2/detection_loss‚0.4667678940782893 epoch‚45 .loss/FRt1/detection_loss‚0.511102000006499 &loss/total_loss‚1.8065527080642365 trainer/global_step‚18249 % _timestamp‚1.7866634940798852e+09 _runtime‚4842.609693962 _step‚1504 1loss/FRt1/assignment_loss‚0.46073244484811304°¬pHzÏ*ô epoch‚45 .loss/FRt1/detection_loss‚0.511102000006499 &loss/total_loss‚1.8065527080642365 trainer/global_step‚18249 % _timestamp‚1.7866634940798852e+09 _runtime‚4842.609693962 _step‚1504 1loss/FRt1/assignment_loss‚0.46073244484811304 0loss/FRt2/assignment_loss‚0.3679503691313353 /loss/FRt2/detection_loss‚0.4667678940782893à ÷±æ˜”Ð*‚(:‰ ȤùÓ¢ ™á gpu.0.gpu‚3.0 gpu.0.memory‚0, gpu.0.memoryAllocated‚7.0313262939453125) gpu.0.memoryAllocatedBytes‚ 6039863296 gpu.0.temp‚35.0 gpu.0.powerWatts‚206.144' gpu.0.enforcedPowerLimitWatts‚500.0 gpu.0.powerPercent‚41.2288 gpu.0.smClock‚1410 gpu.0.memoryClock‚1593$ gpu.0.correctedMemoryErrors‚1955# gpu.0.uncorrectedMemoryErrors‚0¶#~‹Ñ*‚(:€ ȤùÓ£ëîã disk.sda1.in‚0 disk.sda2.in‚0 disk.sda2.out‚0& disk.nvme0n1.out‚63063.90185546875) disk./.usagePercent‚32.76791987201753 disk.sda3.in‚ 44704.5859375 disk.sda.out‚89.5 disk.dm-1.in‚ 44704.5859375 disk.drbd2165.in‚0 disk.dm-0.in‚70136.08984375# disk.dm-0.out‚63063.90185546875) proc.memory.availableMB‚ 5194.70703125% disk./.usageGB‚488.26568603515625( disk.nvme0c0n1.out‚63063.90185546875 disk.dm-1.out‚89.5 disk.drbd2165.out‚0 cpu‚27.253867389644295 disk.sda3.out‚89.5 disk.sda1.out‚0" disk.nvme0n1.in‚70136.08984375$ memory_percent‚68.29402446746826" proc.memory.rssMB‚ 2299.5546875* proc.memory.percent‚14.035367965698242 proc.cpu.threads‚17 network.sent‚ 2.3742356e+07 network.recv‚ 9.117486e+06$ disk.nvme0c0n1.in‚70136.08984375 disk.sda.in‚ 44704.5859375xº-2Ò*- _wandbruntime‚4854  _runtime‚4854Ü®è|ËÓ*Å ' lr-AdamW/pg1‚2.7703557254298907e-05 ' lr-AdamW/pg2‚2.7703557254298907e-05 trainer/global_step‚18299 % _timestamp‚1.7866635060986805e+09 _runtime‚4854.630865459 _step‚1505üd²ÓÐÔ*Ê trainer/global_step‚18299 % _timestamp‚1.7866635060986805e+09 _runtime‚4854.630865459 _step‚1505 ' lr-AdamW/pg1‚2.7703557254298907e-05 ' lr-AdamW/pg2‚2.7703557254298907e-05á ƒ{G2Õ*- _wandbruntime‚4854  _runtime‚4854e@‘Å'Ö*¿O TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚41945088 PIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚69 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚435247999418368 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚499301536628736 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚499301536628736 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 4423991442432 JDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 5368709120 JDeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4289 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚4287 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚61423770 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 @7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚5264 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 4552982528 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚223 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚15352909 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚61478912 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚262 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚4287 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚496350589190144 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 122306526 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 125829120 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚322 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚322 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 4423949497344 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 4552982528 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚ 129177448 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚15352978 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 4678760972288 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 BDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 G>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4876 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 122905600 KBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚4876 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚61428057 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚429 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 831258624 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 5368709120 C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚4289 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 PCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚58387380 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 4414737413733 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 5494538240 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚57532947 UHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚64681984 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚61428057 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚67749391 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 4517085184 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 125829120 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚322 KDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚60 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚67749391 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚499301553733632 D;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚5264 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 % _timestamp‚1.7866635060993404e+09 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 4654602240 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 122905600 IBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚71 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚42179967 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 5368709120 IBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚60 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 4423991442432 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚439926825072640 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 _runtime‚4854.630945741 M@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚59049984 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17104896 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚503725486126080 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚67749389 JBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚665 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚262 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚61423770 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 4423949497344 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚57532876 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 4678805010432 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚209 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚439926760390656 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚435248020062208 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 5494538240 PCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚59049984 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚503725486126080 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 845178880 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 4628521206 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 5494538240 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 4414778761753 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0F¤„)×*ÄO R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚503725486126080 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚67749389 JBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚665 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚262 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚61423770 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 4423949497344 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚57532876 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 4678805010432 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚209 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚439926760390656 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚435248020062208 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 5494538240 PCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚59049984 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚503725486126080 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 845178880 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 4628521206 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 5494538240 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 4414778761753 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚41945088 PIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚69 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚435247999418368 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚499301536628736 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚499301536628736 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 4423991442432 JDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 5368709120 JDeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4289 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚4287 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚61423770 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 @7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚5264 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 4552982528 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚223 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚15352909 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚61478912 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚262 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚4287 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚496350589190144 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 122306526 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 125829120 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚322 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚322 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 4423949497344 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 4552982528 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚ 129177448 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚15352978 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 4678760972288 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 BDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 G>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4876 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 122905600 KBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚4876 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚61428057 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚429 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 831258624 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 5368709120 C:DeviceStatsMonitor.on_train_batch_sta"É°¡ rt/active.all.current‚4289 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 PCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚58387380 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 4414737413733 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 5494538240 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚57532947 UHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚64681984 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚61428057 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚67749391 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 4517085184 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 125829120 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚322 KDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚60 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚67749391 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚499301553733632 D;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚5264 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 % _timestamp‚1.7866635060993404e+09 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 4654602240 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 122905600 IBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚71 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚42179967 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 5368709120 IBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚60 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 4423991442432 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚439926825072640 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 _runtime‚4854.630945741 M@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚59049984 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17104896â  c2Ø*- _wandbruntime‚4854  _runtime‚4854AâˆÏ&Ù*ÉM XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 4414967986345 IBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚60 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚41948160 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 4679050890240 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚48229376 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚322 G@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚60 EDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚322 BDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 831258624 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚61427057 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 4424180566016 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚67752944 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 4415009334413 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚61431350 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17104896 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚439950049452032 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 4552982528 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚42182174 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 125829120 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚262 DDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 129184294 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚439950118325248 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 K>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59053056 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚4295 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚429 % _timestamp‚1.7866635062598667e+09 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚499327858966528 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17104896 NGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚73 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521206 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 122306526 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚ 129184294 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚61431350 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚496376792023040 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 5368709120 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 5368709120 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚67752942 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 5494538240 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 845178880 NADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚58387428 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 5368709120 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 125829120 trainer/global_step‚18299 B9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚5264 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚223 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚435271067435008 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚61478912 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚500791742970025 E6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚322 D=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚60 >5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5264 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚67752942 H@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚665 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚57535895 SFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚68873216 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚20643840 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 4679002660864 I@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚4876 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚57535970 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚15353723 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚499327876071424 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654602240 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 4424180566016 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚262 P6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚322 D=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚60 >5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5264 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚67752942 H@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚665 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚57535895 SFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚68873216 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚20643840 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 4679002660864 I@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚4876 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚57535970 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚15353723 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚499327876071424 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654602240 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 4424180566016 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚262 PDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚322 BDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 831258624 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚61427057 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 4424180566016 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚67752944 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 4415009334413 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚61431350 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17104896 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚439950049452032 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 4552982528 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚42182174 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 125829120 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚262 DDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 129184294 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚439950118325248 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 K>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59053056 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚4295 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚429 % _timestamp‚1.7866635062598667e+09 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚499327858966528 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17104896 NGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚73 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521206 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 122306526 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚ 129184294 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚61431350 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚496376792023040 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 5368709120 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 5368709120 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚67752942 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 5494538240ã uø’)2Û*- _wandbruntime‚4854  _runtime‚4854(!ŽNrÜ*ì .loss/FRt1/detection_loss‚0.514274205318196 %loss/total_loss‚1.791443571894029 epoch‚45 $ _timestamp‚1.786663506260816e+09 _step‚1508 1loss/FRt1/assignment_loss‚0.46116448593961684 0loss/FRt2/assignment_loss‚0.3837136805927341 /loss/FRt2/detection_loss‚0.4322912000434821 trainer/global_step‚18299 _runtime‚ 4854.79062002¤OÆwÝ*ñ _step‚1508 1loss/FRt1/assignment_loss‚0.46116448593961684 0loss/FRt2/assignment_loss‚0.3837136805927341 /loss/FRt2/detection_loss‚0.4322912000434821 trainer/global_step‚18299 _runtime‚ 4854.79062002 .loss/FRt1/detection_loss‚0.514274205318196 %loss/total_loss‚1.791443571894029 epoch‚45 $ _timestamp‚1.786663506260816e+09ä M|ÿj¡Þ*‚(:– פùÓƒíìà gpu.0.gpu‚64.0 gpu.0.memory‚12, gpu.0.memoryAllocated‚7.0313262939453125) gpu.0.memoryAllocatedBytes‚ 6039863296 gpu.0.temp‚34.0 gpu.0.powerWatts‚124.524' gpu.0.enforcedPowerLimitWatts‚500.0) gpu.0.powerPercent‚24.904799999999998 gpu.0.smClock‚1410 gpu.0.memoryClock‚1593$ gpu.0.correctedMemoryErrors‚1955# gpu.0.uncorrectedMemoryErrors‚0ºÊóà‚ß*‚(:÷ פùÓ»ŒÎâ network.sent‚ 2.3824019e+07% disk./.usageGB‚488.26365661621094 disk.sda2.out‚0 disk.drbd2165.in‚0 disk.drbd2165.out‚0 disk.sda.in‚ 44704.5859375# memory_percent‚68.2917833328247* proc.memory.percent‚14.035367965698242 network.recv‚ 9.135087e+06" disk.nvme0n1.in‚70136.08984375 disk.dm-1.in‚ 44704.5859375 disk.dm-0.in‚70136.08984375" disk.dm-0.out‚63072.0400390625 disk.sda3.out‚90 disk.sda1.out‚0" proc.memory.rssMB‚ 2299.5546875* disk./.usagePercent‚32.767783676009316 disk.sda2.in‚0% disk.nvme0n1.out‚63072.0400390625$ disk.nvme0c0n1.in‚70136.08984375' disk.nvme0c0n1.out‚63072.0400390625 disk.sda.out‚90) proc.memory.availableMB‚ 5195.07421875 cpu‚27.253461641486094 disk.dm-1.out‚90 disk.sda3.in‚ 44704.5859375 disk.sda1.in‚0 proc.cpu.threads‚17YF®Ä2à*- _wandbruntime‚4866  _runtime‚4866ìÚñkËá*Å trainer/global_step‚18349 % _timestamp‚1.7866635183358355e+09 _runtime‚4866.867815445 _step‚1509 ' lr-AdamW/pg1‚2.6108208955178987e-05 ' lr-AdamW/pg2‚2.6108208955178987e-05go:ƒÐâ*Ê trainer/global_step‚18349 % _timestamp‚1.7866635183358355e+09 _runtime‚4866.867815445 _step‚1509 ' lr-AdamW/pg1‚2.6108208955178987e-05 ' lr-AdamW/pg2‚2.6108208955178987e-05å -17Ž2ã*- _wandbruntime‚4866  _runtime‚48661pÄLëä*¿O WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚41348020 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚505053189781504 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 122905600 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 5494538240 IBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚60 D;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚5264 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚61593159 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚223 PIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚67 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 831258624 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚48232448 _runtime‚4866.867925624 UHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚68876288 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚502086178091285 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 5494538240 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 125829120 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 4654602240 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚42290217 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 4435578208256 KBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚4876 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚15393395 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚41945088 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚61478912 M@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚59049984 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17104896 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚57683681 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚67927041 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚322 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚322 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚262 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚4287 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚ 129520200 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 122306526 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 5494538240 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 5368709120 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚262 KDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚60 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚322 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚61593159 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4289 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚15393462 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚20643840 BDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚500617653518336 JDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚505053189781504 PCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚59049984 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 4691086963200 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚500617670623232 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 4435536263168 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚61588872 C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚4289 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚57683612 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 G>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4876 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 4435578208256 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚322 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 YEDÉ¯Ú eviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚502086236478665 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚441091406789120 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 5368709120 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 125829120 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚41945088 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚4287 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚429 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 4426299224341 _step‚1510 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚500617670623232 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 4691038730752 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚505053248831488 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚67927041 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 4517085184 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 4552982528 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚500617653518336 JBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚665 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 5368709120 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 4654602240 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 122905600 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚67927039 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚436400368058368 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚497659878866944;>Ê'å*ÄO UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚505053189781504 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 122905600 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 5494538240 IBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚60 D;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚5264 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚61593159 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚223 PIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚67 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 831258624 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚48232448 _runtime‚4866.867925624 UHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚68876288 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚502086178091285 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 5494538240 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 125829120 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 4654602240 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚42290217 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 4435578208256 KBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚4876 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚15393395 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚41945088 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚61478912 M@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚59049984 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17104896 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚57683681 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚67927041 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚322 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚322 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚262 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚4287 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚ 129520200 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 122306526 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 5494538240 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 5368709120 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚262 KDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚60 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚322 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚61593159 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4289 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚15393462 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚20643840 BDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚500617653518336 JDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚505053189781504 PCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚59049984 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 4691086963200 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚500617670623232 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 4435536263168 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚61588872 C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚4289 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚57683612 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 G>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4876 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 4435578208256 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚322 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚502086236478665 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚441091406789120 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 5368709120 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 125829120 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚41945088 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚4287 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚429 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 4426299224341 _step‚1510 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚500617670623232 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 4691038730752 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚505053248831488 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚67927041 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 4517085184 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 4552982528 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚500617653518336 JBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚665 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 5368709120 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 4654602240 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 122905600 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚67927039 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚436400368058368 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚497659878866944 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚41348020æ a£üC2æ*- _wandbruntime‚4867  _runtime‚4867SŸÈgÎ&ç*ÈM TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 4552982528 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚ 129522751 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 125829120 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚57686597 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 5368709120 D=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚60 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 122905600 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 4691331630592 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚209 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚505079802241024 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654602240 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚57686665 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 4517085184 _step‚1511 EDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 122306526 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚262 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 trainer/global_step‚18349 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚42292424 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚ 129527046 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚61478912 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 122905600 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 831258624 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚497686081699840 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚322 % _timestamp‚1.7866635185058525e+09 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚4295 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚67930594 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚4293 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 4654602240 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 4435809280000 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚41948160 G@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚68 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚15394241 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚436423415431168 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚502112594457433 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚41348068 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521206 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚262 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚502112652844861 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 4426529796953 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 125829120 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚505079743187968 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚41948160 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 845178880 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚436423436075008 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚67930592 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 4435767331840 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚61592159 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚4293 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 5368709120 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 4426571145021 IBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚60 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚322 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 _runtime‚ 4867.03619269 K>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59053056 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚500643992961024 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17104896 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚20643840 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚500643975856128 E5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5264 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17104896 SFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚66776064 NADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚58387428 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 5494538240 PDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 129527046 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚61596452 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚61592159 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 5494538240 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚505079802241024 D6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚322 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 _runtime‚ 4867.03619269 K>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59053056 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚500643992961024 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17104896 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚20643840 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚500643975856128 E5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5264 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17104896 SFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚66776064 NADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚58387428 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 5494538240 PDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 129527046 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚61596452 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚61592159 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 5494538240 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚505079802241024 DDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚57686597 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 5368709120 D=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚60 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 122905600 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 4691331630592 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚209 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚505079802241024 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654602240 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚57686665 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 4517085184 _step‚1511 EDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 122306526 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚262 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 trainer/global_step‚18349 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚42292424 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚ 129527046 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚61478912 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 122905600 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 831258624 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚497686081699840 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚322 % _timestamp‚1.7866635185058525e+09 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚4295 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚67930594 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚4293 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 4654602240 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 4435809280000 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚41948160 G@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚68 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚15394241 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚436423415431168 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚502112594457433 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚41348068 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521206 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚262 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚502112652844861 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 4426529796953 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 125829120 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚505079743187968 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚41948160 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 845178880 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚436423436075008 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0ç „qø2é*- _wandbruntime‚4867  _runtime‚4867id&ê*î 1loss/FRt1/assignment_loss‚/ĆòN0.44971789475985663 epoch‚45 _runtime‚ 4867.03675905 _step‚1512 0loss/FRt2/assignment_loss‚0.3742694914986837 /loss/FRt1/detection_loss‚0.4972096907611563 0loss/FRt2/detection_loss‚0.47668331318203855 %loss/total_loss‚1.797880390201735 trainer/global_step‚18349 $ _timestamp‚1.786663518506946e+09;„é yë*ó 0loss/FRt2/detection_loss‚0.47668331318203855 %loss/total_loss‚1.797880390201735 trainer/global_step‚18349 $ _timestamp‚1.786663518506946e+09 1loss/FRt1/assignment_loss‚0.44971789475985663 epoch‚45 _runtime‚ 4867.03675905 _step‚1512 0loss/FRt2/assignment_loss‚0.3742694914986837 /loss/FRt1/detection_loss‚0.4972096907611563è ŠË8Ü•ì*‚(:Š æ¤ùÓž±œá gpu.0.gpu‚64.0 gpu.0.memory‚12, gpu.0.memoryAllocated‚7.0313262939453125) gpu.0.memoryAllocatedBytes‚ 6039863296 gpu.0.temp‚35.0 gpu.0.powerWatts‚213.735' gpu.0.enforcedPowerLimitWatts‚500.0 gpu.0.powerPercent‚42.747 gpu.0.smClock‚1410 gpu.0.memoryClock‚1593$ gpu.0.correctedMemoryErrors‚1955# gpu.0.uncorrectedMemoryErrors‚0EÔ¹Ví*‚(:’ æ¤ùÓëòã disk.sda.in‚ 44704.5859375 disk.sda1.out‚0 disk.sda2.in‚0 disk.dm-1.out‚ 90.21484375' proc.memory.availableMB‚ 5194.796875* proc.memory.percent‚14.035367965698242 network.recv‚ 9.158835e+06) disk./.usagePercent‚32.76792601619836 disk.sda3.in‚ 44704.5859375$ disk.nvme0c0n1.in‚70136.08984375( disk.nvme0c0n1.out‚63077.64404296875 disk.sda.out‚ 90.21484375 disk.sda2.out‚0& disk.nvme0n1.out‚63077.64404296875 disk.sda1.in‚0 disk.dm-1.in‚ 44704.5859375 disk.drbd2165.in‚0 disk.drbd2165.out‚0 disk.dm-0.in‚70136.08984375# disk.dm-0.out‚63077.64404296875$ memory_percent‚68.29347610473633" proc.memory.rssMB‚ 2299.5546875 disk.sda3.out‚ 90.21484375" disk.nvme0n1.in‚70136.08984375 cpu‚27.253080005840136 proc.cpu.threads‚17 network.sent‚ 2.3891498e+07$ disk./.usageGB‚488.2657775878906Àšš2î*- _wandbruntime‚4879  _runtime‚4879œ§Ü§Êï*Ä $ _timestamp‚1.786663530769138e+09 _runtime‚4879.299157437 _step‚1513 ' lr-AdamW/pg1‚2.4559354873095797e-05 ' lr-AdamW/pg2‚2.4559354873095797e-05 trainer/global_step‚18399Ÿ XÏð*É ' lr-AdamW/pg1‚2.4559354873095797e-05 ' lr-AdamW/pg2‚2.4559354873095797e-05 trainer/global_step‚18399 $ _timestamp‚1.786663530769138e+09 _runtime‚4879.299157437 _step‚1513é Xj¨¹2ñ*- _wandbruntime‚4879  _runtime‚4879yåD5Å'ò*¿O \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚498969168543744 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 125829120 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 4552982528 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚442256040214016 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚ 129858663 IBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚74 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 125829120 F?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚60 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚506380952486912 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚262 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚57834189 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚42400467 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚262 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚501933770407936 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚41945088 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚20643840 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 4552982528 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚61478912 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚68104689 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 122905600 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚501933787512832 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 845178880 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 4654602240 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚506380952486912 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚501933787512832 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 4437861034949 KDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚60 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 122905600 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚322 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚429 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚209 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚48232448 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 5494538240 IBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚60 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 125829120 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚61753974 G>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4876 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 4447164974080 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚498969185583104 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 4437902382969 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚262 trainer/global_step‚18399 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚501933770407936 @7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚5264 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚322 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚41945088 JDeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4289 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚437552757342208 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 5368709120 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚68104691 JBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚665 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 M@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚59049984 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 4447164974080 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚61758261 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚15433720 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚442256109090304 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚41348020 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚322 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚57834115 UHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚68876288 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚503407087966073 % _timestamp‚1.7866635307700646e+09 C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚4289 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚506380893436928 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17104896 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 4703351748096 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚503407029578693 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 4654602240 D;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚5264 PCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚58387380 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 PIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚72 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 5494538240 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚506380893436928 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚61753974 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 _runtime‚4879.303888247 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚223 JDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚61758261 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 4703303515648 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 4517085184 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚42400469 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚4287 KBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚4876 BDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚506380893436928 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚61753974 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 _runtime‚4879.303888247 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚223 JDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚61758261 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 4703303515648 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 4517085184 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚42400469 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚4287 KBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚4876 BDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4876 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 4447164974080 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚498969185583104 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 4437902382969 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚262 trainer/global_step‚18399 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚501933770407936 @7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚5264 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚322 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚41945088 JDeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4289 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚437552757342208 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 5368709120 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚68104691 JBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚665 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 M@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚59049984 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 4447164974080 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚61758261 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚15433720 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚442256109090304 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚41348020 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚322 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚57834115 UHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚68876288 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚503407087966073 % _timestamp‚1.7866635307700646e+09 C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚4289 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚506380893436928 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17104896 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 4703351748096 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚503407029578693 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 4654602240 D;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚5264ê Äóf2ô*- _wandbruntime‚4879  _runtime‚4879qViõ%õ*ÈM >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚68108242 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚322 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 122905600 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 4438132955629 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚322 >5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5264 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 4447354097664 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 NADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚58387428 D=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚60 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚68108244 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 4447354097664 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚68108244 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚209 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚503433445944841 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚4295 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚506407505896448 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚506407446843392 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 122306526 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚501960109850624 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚61757261 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚498995354337280 G@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚60 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 125829120 K>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59053056 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 4447396045824 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚498995371376640 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 5494538240 PDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654602240 G@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚67 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521206 % _timestamp‚1.7866635309310617e+09 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17104896 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚501960092745728 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 831258624 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚503433504332269 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚57837135 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚223 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 4438091607561 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 5494538240 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚ 129865503 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚20643840 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚41348068 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 5368709120 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚437575804715008 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 4703597824512 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 4517085184 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚262 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17104896 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚501960092745728 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚442279402539520 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚61478912 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 NGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚65 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 5368709120 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚262 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚41948160 NADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚59053056 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚15434528 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 _step‚1515 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚501960109850624 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚57837202 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚15434463 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚322 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 125829120 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚4293 SFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚68873216 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚61757261 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚41948160 B9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚5264 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 4703549595136 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 BDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 129869798 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚42402672 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚437575784071168 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 5368709120 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 125829120 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 4654602240 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚68108242 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚61761554 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 4552982528 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚429 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚48229376 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚262 E6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚322 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 125829120 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚4293 SFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚68873216 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚61757261 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚41948160 B9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚5264 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 4703549595136 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 BDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 129869798 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚42402672 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚437575784071168 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 5368709120 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 125829120 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 4654602240 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚68108242 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚61761554 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 4552982528 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚429 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚48229376 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚262 E8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚68108242 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚322 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 122905600 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 4438132955629 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚322 >5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5264 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 4447354097664 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 NADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚58387428 D=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚60 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚68108244 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 4447354097664 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚68108244 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚209 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚503433445944841 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚4295 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚506407505896448 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚506407446843392 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 122306526 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚501960109850624 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚61757261 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚498995354337280 G@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚60 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 125829120 K>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59053056 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 4447396045824 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚498995371376640 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 5494538240 PDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654602240 G@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚67 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521206 % _timestamp‚1.7866635309310617e+09 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17104896 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚501960092745728 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 831258624 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚503433504332269 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚57837135 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚223 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 4438091607561 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 5494538240 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚ 129865503 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚20643840 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚41348068 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 5368709120 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚437575804715008 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 4703597824512 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 4517085184 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚262 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17104896 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚501960092745728 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚442279402539520 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚61478912 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 NGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚65 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 5368709120 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚262 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚41948160 NADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚59053056 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚15434528 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 _step‚1515ë °„š,2÷*- _wandbruntime‚4879  _runtime‚4879þ`5uø*ï 1loss/FRt2/assignment_loss‚0.39151525266040677 /loss/FRt1/detection_loss‚0.4667924723724461 &loss/total_loss‚1.7691658598465012 epoch‚45 trainer/global_step‚18399 $ _timestamp‚1.786663530932132e+09 _runtime‚4879.461977484 0loss/FRt1/assignment_loss‚0.4446106035234977 /loss/FRt2/detection_loss‚0.4662475312901507 _step‚1516ŽÜ—Œzù*ô 1loss/FRt2/assignment_loss‚0.39151525266040677 /loss/FRt1/detection_loss‚0.4667924723724461 &loss/total_loss‚1.7691658598465012 epoch‚45 trainer/global_step‚18399 $ _timestamp‚1.786663530932132e+09 _runtime‚4879.461977484 0loss/FRt1/assignment_loss‚0.4446106035234977 /loss/FRt2/detection_loss‚0.4662475312901507 _step‚1516ì Ù»Q2ú*- _wandbruntime‚4884  _runtime‚4884G,Ãâßû*Ù 3Purity/1FRt/event_proportion‚0.6297432780265808 2Purity/2FRt/event_proportion‚0.040062315762043 'particle/accuracy‚0.639288067817688 'particle/f_score‚0.5542874336242676 0Purity/2FRt/event_purity‚0.43222764134407043 $ _timestamp‚1.786663535928657e+09 _step‚1517 /Purity/1FRt/event_purity‚0.4682350754737854 -Purity/1FRt/FRt_purity‚0.4682350754737854 .particle/accuracy_1_of_2‚0.923172652721405 +validation_accuracy‚0.21285554766654968 0Purity/*FRt/event_purity‚0.46620655059814453 2Purity/*FRt/event_proportion‚0.669805645942688 -Purity/*FRt/FRt_purity‚0.4796169400215149 *jet/accuracy_1_of_2‚0.5774663090705872 particle/accuracy_1_of_1‚1 *loss/val_total_loss‚2.5378409249355194 +jet/accuracy_2_of_2‚0.21285554766654968 +particle/specificity‚0.6432937979698181 _runtime‚4884.458686884 Purity/0FRt/event_purity‚1 +jet/accuracy_1_of_1‚0.30944371223449707 +particle/sensitivity‚0.6319720149040222 epoch‚45 3Purity/0FRt/event_proportion‚0.3301943838596344 -Purity/2FRt/FRt_purity‚0.5676093697547913 /particle/accuracy_2_of_2‚0.6835886836051941 7validation_average_jet_accuracy‚0.31461310386657715 trainer/global_step‚18399JŽäü*Þ 0Purity/2FRt/event_purity‚0.43222764134407043 $ _timestamp‚1.786663535928657e+09 _step‚1517 /Purity/1FRt/event_purity‚0.4682350754737854 -Purity/1FRt/FRt_purity‚0.4682350754737854 .particle/accuracy_1_of_2‚0.923172652721405 +validation_accuracy‚0.21285554766654968 0Purity/*FRt/event_purity‚0.46620655059814453 2Purity/*FRt/event_proportion‚0.669805645942688 -Purity/*FRt/FRt_purity‚0.4796169400215149 *jet/accuracy_1_of_2‚0.5774663090705872 particle/accuracy_1_of_1‚1 *loss/val_total_loss‚2.5378409249355194 +jet/accuracy_2_of_2‚0.21285554766654968 +particle/specificity‚0.6432937979698181 _runtime‚4884.458686884 Purity/0FRt/event_purity‚1 +jet/accuracy_1_of_1‚0.30944371223449707 +particle/sensitivity‚0.6319720149040222 epoch‚45 3Purity/0FRt/event_proportion‚0.3301943838596344 -Purity/2FRt/FRt_purity‚0.5676093697547913 /particle/accuracy_2_of_2‚0.6835886836051941 7validation_average_jet_accuracy‚0.31461310386657715 trainer/global_step‚18399 3Purity/1FRt/event_proportion‚0.6297432780265808 2Purity/2FRt/event_proportion‚0.040062315762043 'particle/accuracy‚0.639288067817688 'particle/f_score‚0.5542874336242676í ··¨î¡ý*‚(:– õ¤ùÓ«„£á gpu.0.gpu‚64.0 gpu.0.memory‚12, gpu.0.memoryAllocated‚7.0313262939453125) gpu.0.memoryAllocatedBytes‚ 6039863296 gpu.0.temp‚35.0 gpu.0.powerWatts‚129.345' gpu.0.enforcedPowerLimitWatts‚500.0) gpu.0.powerPercent‚25.868999999999996 gpu.0.smClock‚1410 gpu.0.memoryClock‚1593$ gpu.0.correctedMemoryErrors‚1955# gpu.0.uncorrectedMemoryErrors‚0±3!á‡þ*‚(:ü õ¤ùÓ¬©ýâ" disk.nvme0n1.in‚70136.08984375& disk.nvme0n1.out‚63085.94482421875( proc.memory.availableMB‚ 5155.6015625 cpu‚27.2580028482806 network.recv‚ 9.187425e+06 disk.dm-1.out‚90.5 disk.drbd2165.out‚0# disk.dm-0.out‚63085.94482421875 disk.sda3.out‚90.5$ disk.nvme0c0n1.in‚70136.08984375( disk.nvme0c0n1.out‚63085.94482421875 disk.sda.in‚ 44704.5859375 network.sent‚ 2.3962772e+07 disk.drbd2165.in‚0 disk.dm-0.in‚70136.08984375 disk.sda3.in‚ 44704.5859375 disk.sda1.out‚0 disk.sda2.out‚0$ memory_percent‚68.53270530700684# proc.memory.rssMB‚ 2338.62109375 disk.sda1.in‚0 disk.sda2.in‚0* proc.memory.percent‚14.273810386657715 proc.cpu.threads‚17( disk./.usagePercent‚32.7679098877237$ disk./.usageGB‚488.2655372619629 disk.dm-1.in‚ 44704.5859375 disk.sda.out‚90.5 ¥,2ÿ*- _wandbruntime‚4896  _runtime‚4896:)”ÌË€+Å trainer/global_step‚18449 % _timestamp‚1.7866635479361022e+09 _runtime‚4896.466114468 _step‚1518 ' lr-AdamW/pg1‚2.3057094488052737e-05 ' lr-AdamW/pg2‚2.3057094488052737e-05Ï^jÐ+Ê trainer/global_step‚18449 % _timestamp‚1.7866635479361022e+09 _runtime‚4896.466114468 _step‚1518 ' lr-AdamW/pg1‚2.3057094488052737e-05 ' lr-AdamW/pg2‚2.3057094488052737e-05î ¤12‚+- _wandbruntime‚4896  _runtime‚4896×ÞŒÅ'ƒ+¿O `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚500659374751744 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 KBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚4876 C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚4289 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚61941263 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 4462819463680 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚322 IBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚60 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚58034744 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17104896 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚41945088 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚ 130288497 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚505112904917422 KDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚60 JBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚665 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚58034833 IBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚89 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 4517085184 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚322 _runtime‚4896.470867271 F>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 UHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚68876288 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 122306526 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚443724125830656 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚439004375646208 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚48232448 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 PCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚58387380 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚41348020 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚508103661081088 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚508103661081088 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚508103602031104 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚443724056954368 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚439004396290048 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 5494538240 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚503640841617408 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚68342947 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚15483413 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 845178880 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚503640841617408 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚503640824512512 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚209 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚61478912 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 4628521206 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚262 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚61945550 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚68342947 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 4552982528 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚68342945 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 125829120 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 trainer/global_step‚18449 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17104896 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 125829120 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚61941263 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚223 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 4453530165678 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚262 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 4654602240 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚61945550 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4289 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 831258624 BDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚508103602031104 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚429 _step‚1519 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚20643840 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚505112846530042 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 F?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚60 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚42551331 JDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4876 PCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚59049984 JDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 UHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚68876288 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 122306526 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚443724125830656 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚439004375646208 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚48232448 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 PCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚58387380 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚41348020 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚508103661081088 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚508103661081088 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚508103602031104 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚443724056954368 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚439004396290048 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 5494538240 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚503640841617408 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚68342947 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚15483413 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 845178880 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚503640841617408 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚503640824512512 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚209 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚61478912 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 4628521206 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚262 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚61945550 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚68342947 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 4552982528 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚68342945 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 125829120 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 trainer/global_step‚18449 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17104896 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 125829120 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚61941263 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚223 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 4453530165678 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚262 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 4654602240 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚61945550 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4289 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 831258624 BDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚508103602031104 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚429 _step‚1519 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚20643840 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚505112846530042 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 F?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚60 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚42551331 JDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4876 PCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚59049984 JDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521206 G@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚60 P8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚41348068 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 5494538240 trainer/global_step‚18449 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 4654602240 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17104896 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚223 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚503667146850304 SFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚70970368 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚4293 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 % _timestamp‚1.7866635481010923e+09 _runtime‚4896.631373451 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚61948843 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 122905600 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚68346500 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚68346498 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚42553538 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 122306526 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 4463008587264 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚ 130291048 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚68346500 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚61944550 NGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚78 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚209 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 831258624 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 4719975944704 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 4463050535424 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚42553536 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚61478912 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 4453760738338 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 5368709120 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 4552982528 BDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚322 >5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5264 EDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654602240 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 130295343 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚508130155437568 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚15484281 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 5494538240 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚262 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 4552982528 G@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚80 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 4719925618176 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 5368709120 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 125829120 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚4293 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 122905600 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚508130214490624 NADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚59053056 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚58037817 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚61944550 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚503667163955200 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 DDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 D=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚60 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚322 K>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59053056 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚443747348637184 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚262 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚68346498 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚ 130295343 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚508130214490624 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17104896 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚443747419607552 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚505139262896190 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 125829120 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 4463050535424 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 4463008587264 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚58037897 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 5494538240 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚429 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 5368709120 _step‚1520 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚15484359 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚322Z0ÞÔ&‡+ÎM L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 130295343 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚508130155437568 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚15484281 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 5494538240 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚262 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 4552982528 G@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚80 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 4719925618176 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 5368709120 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 125829120 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚4293 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 122905600 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚508130214490624 NADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚59053056 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚58037817 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚61944550 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚503667163955200 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 DDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 D=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚60 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚322 K>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59053056 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚443747348637184 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚262 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚68346498 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚ 130295343 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚508130214490624 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17104896 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚443747419607552 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚505139262896190 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 125829120 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 4463050535424 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 4463008587264 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚58037897 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 5494538240 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚429 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 5368709120 _step‚1520 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚15484359 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚322 EDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521206 G@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚60 P8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚41348068 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 5494538240 trainer/global_step‚18449 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 4654602240 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17104896 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚223 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚503667146850304 SFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚70970368 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚4293 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 % _timestamp‚1.7866635481010923e+09 _runtime‚4896.631373451 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚61948843 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 122905600 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚68346500 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚68346498 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚42553538 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 122306526 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 4463008587264 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚ 130291048 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚68346500 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚61944550 NGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚78 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚209 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 831258624 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 4719975944704 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 4463050535424 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚42553536 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚61478912 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 4453760738338 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 5368709120 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 4552982528 BDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚322 >5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5264 EDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654602240ð ö‘v2ˆ+- _wandbruntime‚4896  _runtime‚4896#Zu‰+ï _step‚1521 0loss/FRt1/assignment_loss‚0.4569900084254034 1loss/FRt2/assignment_loss‚0.39179356523399544 0loss/FRt1/detection_loss‚0.48979926850083955 /loss/FRt2/detection_loss‚0.4790137773381261 epoch‚46 # _timestamp‚1.78666354810209e+09 _runtime‚4896.631906319 &loss/total_loss‚1.8175966194983646 trainer/global_step‚18449Ï „æzŠ+ô 0loss/FRt1/assignment_loss‚0.4569900084254034 1loss/FRt2/assignment_loss‚0.39179356523399544 0loss/FRt1/detection_loss‚0.48979926850083955 /loss/FRt2/detection_loss‚0.4790137773381261 epoch‚46 # _timestamp‚1.78666354810209e+09 _runtime‚4896.631906319 &loss/total_loss‚1.8175966194983646 trainer/global_step‚18449 _step‚1521ñ L ¤¡‹+‚(:– „¥ùÓÖ³öà gpu.0.gpu‚63.0 gpu.0.memory‚12, gpu.0.memoryAllocated‚7.0313262939453125) gpu.0.memoryAllocatedBytes‚ 6039863296 gpu.0.temp‚35.0 gpu.0.powerWatts‚191.698' gpu.0.enforcedPowerLimitWatts‚500.0) gpu.0.powerPercent‚38.339600000000004 gpu.0.smClock‚1410 gpu.0.memoryClock‚1593$ gpu.0.correctedMemoryErrors‚1955# gpu.0.uncorrectedMemoryErrors‚0ëiZŒ+‚(:’ „¥ùÓdžÑâ# proc.memory.rssMB‚ 2338.62109375* proc.memory.percent‚14.273810386657715 disk.sda.out‚ 90.76171875 disk.sda2.in‚0# disk.dm-0.out‚63089.48876953125 proc.cpu.threads‚17) disk./.usagePercent‚32.69240609766528% disk./.usageGB‚487.14047622680664 disk.sda3.in‚ 44722.5703125$ disk.nvme0c0n1.in‚70146.76953125 disk.sda1.in‚0 disk.sda2.out‚0 disk.drbd2165.in‚0 cpu‚27.25767902653294 network.sent‚ 2.403405e+07 network.recv‚ 9.214048e+06 disk.sda3.out‚ 90.76171875 disk.sda.in‚ 44722.5703125" disk.nvme0n1.in‚70146.76953125 disk.dm-1.in‚ 44722.5703125 disk.dm-1.out‚ 90.76171875$ memory_percent‚68.52941513061523( disk.nvme0c0n1.out‚63089.48876953125 disk.sda1.out‚0& disk.nvme0n1.out‚63089.48876953125 disk.drbd2165.out‚0 disk.dm-0.in‚70146.76953125' proc.memory.availableMB‚ 5156.140625\Qü*2+- _wandbruntime‚4908  _runtime‚4908™Ô5÷ÊŽ+Ä ' lr-AdamW/pg1‚2.1601524287427067e-05 ' lr-AdamW/pg2‚2.1601524287427067e-05 trainer/global_step‚18499 $ _timestamp‚1.786663560156052e+09 _runtime‚4908.687910559 _step‚1522ãÈÝNÏ+É ' lr-AdamW/pg1‚2.1601524287427067e-05 ' lr-AdamW/pg2‚2.1601524287427067e-05 trainer/global_step‚18499 $ _timestamp‚1.786663560156052e+09 _runtime‚4908.687910559 _step‚1522ò £yÌ2+- _wandbruntime‚4908  _runtime‚4908ó){…‘+¿O IBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚75 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚42661581 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 4654602240 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚504956958507008 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚41945088 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4289 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚4287 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚440156744286208 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 5368709120 PCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚59049984 D;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚5264 KBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚4876 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚209 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚506433756404830 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 5368709120 PIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚73 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 _step‚1523 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚429 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 IBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚60 _runtime‚4908.688002863 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚68520597 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚62110652 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚62106365 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚506433698017450 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚ 130626960 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚68520595 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚20643840 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚4287 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚58186122 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 JDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 PCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚58387380 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚62110652 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 125829120 F?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚60 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚223 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚509431364736512 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 4474406229504 @7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚5264 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 4552982528 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚15524539 BDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4876 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚509431305686528;k(8Ê'’+ÄO G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚429 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 IBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚60 _runtime‚4908.688002863 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚68520597 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚62110652 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚62106365 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚506433698017450 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚ 130626960 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚68520595 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚20643840 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚4287 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚58186122 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 JDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 PCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚58387380 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚62110652 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 125829120 F?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚60 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚223 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚509431364736512 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 4474406229504 @7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚5264 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 4552982528 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚15524539 BDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4876 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚509431305686528 IBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚75 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚42661581 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 4654602240 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚504956958507008 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚41945088 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4289 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚4287 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚440156744286208 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 5368709120 PCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚59049984 D;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚5264 KBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚4876 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚209 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚506433756404830 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 5368709120 PIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚73 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 _step‚1523ó ×–†2“+- _wandbruntime‚4908  _runtime‚4908­ëòÏ&”+ÉM A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚4295 D5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5264 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 845178880 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521206 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 5494538240 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚509457859092992 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚41948160 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚440179812302848 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 4732261856768 NADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚58387428 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚504983280844800 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚509457918146048 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚262 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 4654602240 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚20643840 trainer/global_step‚18499 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚ 130633800 NADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚59053056 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚62113945 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚223 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 5494538240 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654602240 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚62109652 PDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 130638095 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚58189025 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚504983263739904 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚429 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 5368709120 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17104896 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚68524150 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚262 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 122905600 I@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚4876 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 4465281200878 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 125829120 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17104896 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚4293 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 4465322548946 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 IBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚60 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚41948160 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 4517085184 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚504983280844800 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 5368709120 E6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚322 G@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚60 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚68524150 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 4474595353088 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚ 130633800 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚58189094 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 H@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚665 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚61478912 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚506460172771026 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 _runtime‚4908.851395865 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 4552982528 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚15525239 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚209 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚444912074159616 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 5494538240 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚322 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚509457918146048 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 4474637301248 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚42663788 K>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59053056 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚444912013675008 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚39840768 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚322 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 125829120 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 % _timestamp‚1.7866635603209844e+09 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚506460114383598 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 4474637301248 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚ 130638095 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚68524148 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 122905600 B9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚5264 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚68524148 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚440179791659008 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 4732222016000 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚41348068 E6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚322 G@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚60 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚68524150 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 4474595353088 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚ 130633800 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚58189094 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 H@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚665 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚61478912 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚506460172771026 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 _runtime‚4908.851395865 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 4552982528 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚15525239 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚209 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚444912074159616 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 5494538240 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚322 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚509457918146048 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 4474637301248 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚42663788 K>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59053056 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚444912013675008 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚39840768 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚322 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 125829120 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 % _timestamp‚1.7866635603209844e+09 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚506460114383598 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 4474637301248 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚ 130638095 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚68524148 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 122905600 B9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚5264 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚68524148 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚440179791659008 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 4732222016000 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚41348068 E5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5264 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 845178880 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521206 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 5494538240 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚509457859092992 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚41948160 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚440179812302848 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 4732261856768 NADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚58387428 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚504983280844800 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚509457918146048 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚262 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 4654602240 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚20643840 trainer/global_step‚18499 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚ 130633800 NADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚59053056 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚62113945 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚223 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 5494538240 M>DeviceStatsMonitor.on_train_batch_end/aóãF£llocated_bytes.all.peak‚ 4654602240 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚62109652 PDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 130638095 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚58189025 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0ô K—=Y2–+- _wandbruntime‚4908  _runtime‚4908gÃ5Îu—+ï 1loss/FRt1/assignment_loss‚0.47151982716432267 /loss/FRt1/detection_loss‚0.4843108044912414 epoch‚46 trainer/global_step‚18499 % _timestamp‚1.7866635603220794e+09 _runtime‚4908.852482088 _step‚1525 1loss/FRt2/assignment_loss‚0.36081977105969926 /loss/FRt2/detection_loss‚0.4597235574984769 $loss/total_loss‚1.77637396021374Ê©®œz˜+ô epoch‚46 trainer/global_step‚18499 % _timestamp‚1.7866635603220794e+09 _runtime‚4908.852482088 _step‚1525 1loss/FRt2/assignment_loss‚0.36081977105969926 /loss/FRt2/detection_loss‚0.4597235574984769 $loss/total_loss‚1.77637396021374 1loss/FRt1/assignment_loss‚0.47151982716432267 /loss/FRt1/detection_loss‚0.4843108044912414õ 7Pd¿–™+‚(:‹ “¥ùÓ·Áúà gpu.0.gpu‚60.0 gpu.0.memory‚11, gpu.0.memoryAllocated‚7.0313262939453125) gpu.0.memoryAllocatedBytes‚ 6039863296 gpu.0.temp‚35.0 gpu.0.powerWatts‚226.829' gpu.0.enforcedPowerLimitWatts‚500.0 gpu.0.powerPercent‚45.3658 gpu.0.smClock‚1410 gpu.0.memoryClock‚1593$ gpu.0.correctedMemoryErrors‚1955# gpu.0.uncorrectedMemoryErrors‚0lé·”š+‚(:‰ “¥ùÓúÉâ disk.drbd2165.in‚0 disk.sda3.in‚ 45492.109375 disk.sda3.out‚ 91.04296875$ disk.nvme0c0n1.in‚70146.76953125 cpu‚27.257152266964756 proc.cpu.threads‚17 network.sent‚ 2.4094448e+07 network.recv‚ 9.230878e+06$ disk./.usageGB‚487.0019416809082" disk.dm-0.out‚63095.5439453125' disk.nvme0c0n1.out‚63095.5439453125 disk.sda2.in‚0 disk.sda2.out‚0" disk.nvme0n1.in‚70146.76953125) disk./.usagePercent‚32.68310892805181 disk.drbd2165.out‚0 disk.sda.in‚ 45492.109375 disk.sda.out‚ 91.04296875% disk.nvme0n1.out‚63095.5439453125# proc.memory.availableMB‚5155.75# proc.memory.rssMB‚ 2338.62109375* proc.memory.percent‚14.273810386657715 disk.dm-0.in‚70146.76953125 disk.sda1.in‚0 disk.sda1.out‚0$ memory_percent‚68.53179931640625 disk.dm-1.in‚ 45492.109375 disk.dm-1.out‚ 91.04296875fUãp2›+- _wandbruntime‚4921  _runtime‚4921ÃÖuVÉœ+à & lr-AdamW/pg1‚2.019273775977293e-05 & lr-AdamW/pg2‚2.019273775977293e-05 trainer/global_step‚18549 % _timestamp‚1.7866635726651475e+09 _runtime‚4921.196892758 _step‚1526}ÐnêÎ+È % _timestamp‚1.7866635726651475e+09 _runtime‚4921.196892758 _step‚1526 & lr-AdamW/pg1‚2.019273775977293e-05 & lr-AdamW/pg2‚2.019273775977293e-05 trainer/global_step‚18549ö úÌH¯2ž+- _wandbruntime‚4921  _runtime‚4921¯ü¬Å'Ÿ+¿O UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 5368709120 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚510759068391936 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚68698247 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚4287 UHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚66779136 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 4552982528 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚68698245 KDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚60 JDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 4654602240 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚42771831 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚322 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚510759009341952 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 122905600 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚223 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚322 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 4476653786894 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚441309133570048 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 BDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4876 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4289 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚446053438243840 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚446053371464704 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 5494538240 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚ 130969712 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 4485992995328 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 122905600 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚429 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚262 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 5368709120 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 IBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚75 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚42771833 JDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚510759009341952 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 4744258538496 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 KBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚4876 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚15565346 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚20643840 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚507754549504858 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚503277937065984 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚322 _step‚1527 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚506273075396608 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 4485951050240 PIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚73 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚62275754 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚507754607892238 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 122306526ÍšøNÊ' +ÄO YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚507754607892238 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 122306526 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 5368709120 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚510759068391936 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚68698247 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚4287 UHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚66779136 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 4552982528 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚68698245 KDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚60 JDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 4654602240 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚42771831 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚322 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚510759009341952 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 122905600 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚223 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚322 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 4476653786894 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚441309133570048 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 BDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4876 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4289 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚446053438243840 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚446053371464704 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 5494538240 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚ 130969712 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 4485992995328 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 122905600 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚429 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚262 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 5368709120 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 IBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚75 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚42771833 JDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚510759009341952 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 4744258538496 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 KBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚4876 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚15565346 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚20643840 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚507754549504858 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚503277937065984 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚322 _step‚1527 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚506273075396608 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 4485951050240 PIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚73 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚62275754 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0÷ Ùtf2¡+- _wandbruntime‚4921  _runtime‚4921ôz¡•#¢+ÉM E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚4293 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚446076724590592 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚506299380629504 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚41948160 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 4486182118912 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 trainer/global_step‚18549 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚42774038 E6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚322 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 125829120 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚503304122859520 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚322 NGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚62 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 122306526 IBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚60 BDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521206 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 _step‚1528 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚ 130976552 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚506299397734400 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚510785621801472 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 P8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 5368709120 D5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5264 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 130980847 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 NADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚59053056 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚68701798 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 122905600 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 4552982528 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚506299380629504 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚322 EDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654602240 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚15566223 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 4486224067072 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚506299397734400 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚503304139898880 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 4476884359554 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 4476843011486 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚62279047 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚441332160299008 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 5494538240 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚58340197 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚507780965871006 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚507781024258434 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 5494538240 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 5494538240 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚68701800 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚62274754 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚20643840 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 4744543647744 _runtime‚4921.358182267 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚510785621801472 K>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59053056 B9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚5264 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚62279047 G@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚64 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚441332180942848 NADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚58387428 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 4486224067072 I@DeviceStatsMonitorÞi:.on_train_batch_end/allocation.small_pool.peak‚4876 SFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚56290304 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚262 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 4552982528 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚42774036 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚446076668300288 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚35646464 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚61478912 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 % _timestamp‚1.7866635728278797e+09øqÎÂÔ&£+ÎM ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚503304122859520 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚322 NGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚62 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 122306526 IBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚60 BDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521206 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 _step‚1528 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚ 130976552 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚506299397734400 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚510785621801472 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 P8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 5368709120 D5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5264 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 130980847 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 NADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚59053056 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚68701798 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 122905600 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 4552982528 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚506299380629504 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚322 EDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654602240 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚15566223 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 4486224067072 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚506299397734400 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚503304139898880 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 4476884359554 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 4476843011486 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚62279047 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚441332160299008 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 5494538240 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚58340197 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚507780965871006 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚507781024258434 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 5494538240 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 5494538240 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚68701800 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚62274754 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚20643840 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 4744543647744 _runtime‚4921.358182267 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚510785621801472 K>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59053056 B9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚5264 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚62279047 G@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚64 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚441332180942848 NADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚58387428 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 4486224067072 I@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚4876 SFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚56290304 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚262 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 4552982528 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚42774036 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚446076668300288 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚35646464 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚61478912 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 % _timestamp‚1.7866635728278797e+09 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚4293 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚446076724590592 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚506299380629504 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚41948160 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 4486182118912 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 trainer/global_step‚18549 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚42774038 E6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚322 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 125829120 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0ø E›ß¹2¤+- _wandbruntime‚4921  _runtime‚4921Ë{ˆv¥+ð trainer/global_step‚18549 _step‚1529 0loss/FRt1/assignment_loss‚0.4796575634975435 /loss/FRt1/detection_loss‚0.4951878873859253 epoch‚46 % _timestamp‚1.7866635728289218e+09 _runtime‚4921.358717549 1loss/FRt2/assignment_loss‚0.32883927038159694 /loss/FRt2/detection_loss‚0.4854696498405548 &loss/total_loss‚1.7891543711056204à ¥{¦+õ 0loss/FRt1/assignment_loss‚0.4796575634975435 /loss/FRt1/detection_loss‚0.4951878873859253 epoch‚46 % _timestamp‚1.7866635728289218e+09 _runtime‚4921.358717549 1loss/FRt2/assignment_loss‚0.32883927038159694 /loss/FRt2/detection_loss‚0.4854696498405548 &loss/total_loss‚1.7891543711056204 trainer/global_step‚18549 _step‚1529ù NIGQ2§+- _wandbruntime‚4933  _runtime‚4933V§)w˨+Å _step‚1530 ' lr-AdamW/pg1‚1.8830825388816252e-05 ' lr-AdamW/pg2‚1.8830825388816252e-05 trainer/global_step‚18599 % _timestamp‚1.7866635850643275e+09 _runtime‚4933.594327297ÿPõ—Щ+Ê ' lr-AdamW/pg2‚1.8830825388816252e-05 trainer/global_step‚18599 % _timestamp‚1.7866635850643275e+09 _runtime‚4933.594327297 _step‚1530 ' lr-AdamW/pg1‚1.8830825388816252e-05ú ̽ /2ª+- _wandbruntime‚4933  _runtime‚4933ÌçEÄ'«+¾O L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 4654602240 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚507589192286208 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 122905600 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚322 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚512086712997376 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚15606104 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚262 KDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚60 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚ 131312464 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚68875895 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚58488185 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 4654602240 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚442461481566208 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚507589192286208 IBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚69 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 125829120 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚512086772047360 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚61478912 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 PCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚59049984 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 4756593567232 BDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4876 D;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚5264 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚62436569 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚507589175181312 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 4497579761152 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 4756549529088 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚4287 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚442461502210048 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 125829120 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 4497579761152 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4289 PIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚67 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚44038144 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 4552982528 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 4497537816064 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚447218031095296 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 125829120 IBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚60 F?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚60 JDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 845178880 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚429 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17104896 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 831258624 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 4628521206 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 5494538240 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 5368709120 _runtime‚4933.598887215 _step‚1531 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 4552982528 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 5494538240 trainer/global_step‚18599 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚41945088 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚322 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚447218095777280 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚41945088 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 JBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚665 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚504587243782144 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 5368709120 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 122905600 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚512086712997376 KBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚4876 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 4488174249482 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 4488215597502 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚507589175181312 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚68875897 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚509075400992266 C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚4289 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 4497537816064 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚ 131316753 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 UHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚64681984 M@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚59049984 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚504587226742784<ÒÁºÉ'¬+ÃO ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚507589192286208 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 122905600 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚322 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚512086712997376 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚15606104 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚262 KDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚60 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚ 131312464 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚68875895 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚58488185 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 4654602240 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚442461481566208 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚507589192286208 IBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚69 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 125829120 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚512086772047360 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚61478912 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 PCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚59049984 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 4756593567232 BDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4876 D;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚5264 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚62436569 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚507589175181312 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 4497579761152 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 4756549529088 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚4287 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚442461502210048 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 125829120 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 4497579761152 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4289 PIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚67 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚44038144 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 4552982528 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 4497537816064 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚447218031095296 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 125829120 IBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚60 F?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚60 JDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 845178880 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚429 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17104896 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 831258624 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 4628521206 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 5494538240 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 5368709120 _runtime‚4933.598887215 _step‚1531 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 4552982528 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 5494538240 trainer/global_step‚18599 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚41945088 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚322 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚447218095777280 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚41945088 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 JBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚665 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚504587243782144 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 5368709120 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 122905600 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚512086712997376 KBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚4876 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 4488174249482 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 4488215597502 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚507589175181312 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚68875897 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚509075400992266 C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚4289 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 4497537816064 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚ 131316753 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 UHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚64681984 M@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚59049984 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚504587226742784 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 4654602240û 7|a52­+- _wandbruntime‚4933  _runtime‚4933“©„¶®+ÇM REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚62444149 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚322 E?DeviceStatsMonitor.on_tr¶ðž&ain_batch_end/active.large_pool.current‚2 DDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654602240 B9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚5264 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17104896 G@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚68 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚504613429575680 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚4293 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚41948160 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚507615514624000 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 4488404822094 _step‚1532 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚58491264 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚512113325456896 G@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚60 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚447241327496704 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚62444149 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚512113325456896 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 4552982528 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚4293 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 845178880 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚68879450 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 4488446170162 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚4295 PDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59053056 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 I@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚4876 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚42884286 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 5368709120 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 4497768884736 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 4497810832896 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚209 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚447241387981312 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚262 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚507615497519104 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚61478912 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚322 _runtime‚ 4933.7567618 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚68879448 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 4756838398464 trainer/global_step‚18599 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 5494538240 >5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5264 EDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚58491196 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚20643840 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚442484528939008 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 831258624 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚509101875745842 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚ 131319304 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 4654602240 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 4756798557696 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 5494538240 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 5368709120 D=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚60 % _timestamp‚1.7866635852264104e+09 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 122905600 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚507615514624000 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚41348068 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚262 EDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521206 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚504613412536320 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 5494538240 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 125829120 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚ 131323599 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚512113266403840 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚68879450 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚223 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚322 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚41948160 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17104896 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 131323599 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚429 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚15606976 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚39840768 NADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚58387428 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚62439856 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 4497768884736 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚68879448 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚442484549582848 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 4497810832896 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚15606910 SFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚60484608 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 4552982528 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 122905600 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚ 131319304 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚507615497519104,žª–Ò&¯+ÌM _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚442484528939008 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 831258624 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚509101875745842 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚ 131319304 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 4654602240 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 4756798557696 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 5494538240 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 5368709120 D=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚60 % _timestamp‚1.7866635852264104e+09 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 122905600 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚507615514624000 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚41348068 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚262 EDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521206 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚504613412536320 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 5494538240 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 125829120 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚ 131323599 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚512113266403840 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚68879450 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚223 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚322 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚41948160 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17104896 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 131323599 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚429 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚15606976 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚39840768 NADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚58387428 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚62439856 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 4497768884736 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚68879448 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚442484549582848 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 4497810832896 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚15606910 SFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚60484608 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 4552982528 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 122905600 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚ 131319304 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚507615497519104 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚62444149 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚322 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 DDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654602240 B9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚5264 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17104896 G@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚68 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚504613429575680 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚4293 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚41948160 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚507615514624000 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 4488404822094 _step‚1532 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚58491264 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚512113325456896 G@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚60 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚447241327496704 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚62444149 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚512113325456896 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 4552982528 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚4293 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 845178880 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚68879450 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 4488446170162 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚4295 PDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59053056 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 I@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚4876 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚42884286 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 5368709120 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 4497768884736 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 4497810832896 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚209 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚447241387981312 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚262 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚507615497519104 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚61478912 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚322 _runtime‚ 4933.7567618 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚68879448 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 4756838398464 trainer/global_step‚18599 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 5494538240 >5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5264 EDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚58491196 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚20643840ü ÈT’Ó2°+- _wandbruntime‚4933  _runtime‚4933mP“iu±+ï _runtime‚ 4933.75726444 _step‚1533 1loss/FRt2/assignment_loss‚0.38189330202246674 /loss/FRt1/detection_loss‚0.4944191004197394 %loss/total_loss‚1.863031378027284 trainer/global_step‚18599 % _timestamp‚1.7866635852273946e+09 0loss/FRt1/assignment_loss‚0.5140164972562816 0loss/FRt2/detection_loss‚0.47270247832879614 epoch‚46û5ÿz²+ô 0loss/FRt1/assignment_loss‚0.5140164972562816 0loss/FRt2/detection_loss‚0.47270247832879614 epoch‚46 _runtime‚ 4933.75726444 _step‚1533 1loss/FRt2/assignment_loss‚0.38189330202246674 /loss/FRt1/detection_loss‚0.4944191004197394 %loss/total_loss‚1.863031378027284 trainer/global_step‚18599 % _timestamp‚1.7866635852273946e+09ý ¼_ í¡³+‚(:– ¢¥ùÓˆ‡óà gpu.0.gpu‚64.0 gpu.0.memory‚12, gpu.0.memoryAllocated‚7.0313262939453125) gpu.0.memoryAllocatedBytes‚ 6039863296 gpu.0.temp‚34.0 gpu.0.powerWatts‚120.097' gpu.0.enforcedPowerLimitWatts‚500.0) gpu.0.powerPercent‚24.019399999999997 gpu.0.smClock‚1410 gpu.0.memoryClock‚1593$ gpu.0.correctedMemoryErrors‚1955# gpu.0.uncorrectedMemoryErrors‚05‚Jì“´+‚(:ˆ ¢¥ùÓžÂ×â cpu‚27.256676817469007 proc.cpu.threads‚17* disk./.usagePercent‚32.689896711814654 disk.sda2.in‚0& disk.nvme0n1.out‚63155.86279296875 disk.dm-1.out‚ 91.25390625 disk.drbd2165.out‚0 disk.sda3.in‚ 45773.875 disk.sda.out‚ 91.25390625) proc.memory.availableMB‚ 5155.19140625 network.sent‚ 2.4158262e+07# disk./.usageGB‚487.103084564209 disk.sda1.out‚0 disk.drbd2165.in‚0 disk.dm-0.in‚70161.53515625 disk.sda3.out‚ 91.25390625# memory_percent‚68.5352087020874# proc.memory.rssMB‚ 2338.62109375 network.recv‚ 9.250497e+06" disk.nvme0n1.in‚70161.53515625 disk.dm-1.in‚ 45773.875 disk.sda.in‚ 45773.875 disk.sda1.in‚0 disk.sda2.out‚0# disk.dm-0.out‚63155.86279296875$ disk.nvme0c0n1.in‚70161.53515625( disk.nvme0c0n1.out‚63155.86279296875* proc.memory.percent‚14.273810386657715+åßG2µ+- _wandbruntime‚4945  _runtime‚4945lÑ˶+Å ' lr-AdamW/pg1‚1.7515874647643982e-05 ' lr-AdamW/pg2‚1.7515874647643982e-05 trainer/global_step‚18649 % _timestamp‚1.7866635973127944e+09 _runtime‚4945.844879208 _step‚1534üåýз+Ê trainer/global_step‚18649 % _timestamp‚1.7866635973127944e+09 _runtime‚4945.844879208 _step‚1534 ' lr-AdamW/pg1‚1.7515874647643982e-05 ' lr-AdamW/pg2‚1.7515874647643982e-05þ ©“92¸+- _wandbruntime‚4945  _runtime‚4945”CBMÅ'¹+¿O WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 4509124581888 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17104896 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 4552982528 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 122905600 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 trainer/global_step‚18649 JDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚513414416652800 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚508905292070912 PCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚58387380 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚69053545 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚4287 PCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚59049984 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚69053545 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚4287 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚61478912 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚62601671 JDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4876 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 4552982528 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚58639652 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚510396310867054 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚513414475702784 IBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚68 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 845178880 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚505896533458944 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚62605958 D;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚5264 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 4628521206 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 4517085184 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 122306526 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚322 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 4509124581888 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚443613850206208 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 125829120 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 5494538240 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚69053547 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚20643840 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 5368709120 IBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚60 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 125829120 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 831258624 % _timestamp‚1.7866635973134515e+09 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚448382750572544 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 125829120 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚4289 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚ 131655216 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚41945088 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 122905600 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 4768833587200 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 5368709120 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚322 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚508905309175808 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚209 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 4654602240 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 4509166526976 KBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚4876 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚42992333 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚322 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚262 F?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚60 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 4509166526976 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4289 JBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚665 PIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚66 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚448382683793408 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 _step‚1535 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚322 KDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚60 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚508905292070912 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚ 131659505 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚58639584 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚223 _runtime‚4945.844958518 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚62605958 UHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚66779136 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚443613870850048 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚262 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚505896516419584 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 @7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚5264 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 4654602240 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚513414416652800 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚62601671 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚262 F>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚46135296 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0éÅ£º+ÄO R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚513414416652800 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 \HDevi!D''ceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚508905292070912 PCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚58387380 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚69053545 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚4287 PCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚59049984 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚69053545 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚4287 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚61478912 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚62601671 JDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4876 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 4552982528 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚58639652 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚510396310867054 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚513414475702784 IBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚68 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 845178880 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚505896533458944 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚62605958 D;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚5264 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 4628521206 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 4517085184 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 122306526 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚322 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 4509124581888 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚443613850206208 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 125829120 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 5494538240 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚69053547 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚20643840 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 5368709120 IBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚60 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 125829120 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 831258624 % _timestamp‚1.7866635973134515e+09 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚448382750572544 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 125829120 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚4289 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚ 131655216 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚41945088 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 122905600 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 4768833587200 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 5368709120 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚322 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚508905309175808 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚209 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 4654602240 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 4509166526976 KBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚4876 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚42992333 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚322 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚262 F?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚60 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 4509166526976 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4289 JBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚665 PIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚66 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚448382683793408 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 _step‚1535 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚322 KDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚60 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚508905292070912 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚ 131659505 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚58639584 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚223 _runtime‚4945.844958518 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚62605958 UHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚66779136 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚443613870850048 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚262 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚505896516419584 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 @7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚5264 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 4654602240 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚513414416652800 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚62601671 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚262 F>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚46135296 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 4509124581888 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17104896 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 4552982528 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 122905600 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 trainer/global_step‚18649 J8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 BDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654602240 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚58642602 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚15648066 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 4769075275776 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚505922719252480 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚4293 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 D=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚60 G@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚65 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 % _timestamp‚1.7866635974737933e+09 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 125829120 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚62609251 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚508931631513600 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17104896 E6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚322 G@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚60 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 4654602240 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚508931614408704 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 4517085184 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 4552982528 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚448405972854784 trainer/global_step‚18649 NADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚59053056 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17104896 I@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚4876 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 831258624 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 5368709120 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 125829120 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 4552982528 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚443636918222848 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 5368709120 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 4509355650560 B9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚5264 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚42994538 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 5368709120 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚322 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚41948160 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 122905600 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 _runtime‚4946.004008556 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚ 131662056 SFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚68873216 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚443636897579008 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚69057100 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚20643840 IBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚60 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 4500007980770 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 5494538240 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚262 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚69057098 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚4293 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚513441029112320 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚58642667 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 5494538240 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚209 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚48229376 >5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5264 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚508931614408704 NADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚58387428 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 5494538240 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚448406041728000 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521206 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚322 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚262 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚4295 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚ 131662056 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚15648129 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 4769123505152 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚510422727233250 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚223 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 122306526 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚262 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 122905600 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚62609251 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚42994536 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 125829120 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚508931631513600 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚429 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚322 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚62604958 K>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59053056 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚69057098 H@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚665 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 4499966632702 PDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 131666351 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚513440970059264 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚61478912 _step‚1536ü>N¹Ô&½+ÎM J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚62604958 K>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59053056 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚69057098 H@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚665 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 4499966632702 PDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 131666351 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚513440970059264 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚61478912 _step‚1536 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚ 131666351 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 BDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654602240 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚58642602 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚15648066 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 4769075275776 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚505922719252480 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚4293 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 D=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚60 G@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚65 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 % _timestamp‚1.7866635974737933e+09 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 125829120 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚62609251 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚508931631513600 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17104896 E6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚322 G@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚60 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 4654602240 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚508931614408704 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 4517085184 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 4552982528 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚448405972854784 trainer/global_step‚18649 NADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚59053056 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17104896 I@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚4876 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 831258624 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 5368709120 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 125829120 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 4552982528 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚443636918222848 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 5368709120 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 4509355650560 B9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚5264 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚42994538 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 5368709120 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚322 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚41948160 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 122905600 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 _runtime‚4946.004008556 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚ 131662056 SFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚68873216 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚443636897579008 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚69057100 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚20643840 IBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚60 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 4500007980770 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 5494538240 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚262 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚69057098 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚4293 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚513441029112320 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚58642667 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 5494538240 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚209 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚48229376 >5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5264 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚508931614408704 NADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚58387428 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 5494538240 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚448406041728000 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521206 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚322 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚262 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚4295 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚ 131662056 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚15648129 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 4769123505152 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚510422727233250 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚223 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 122306526 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚262 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 122905600 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚62609251 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚42994536 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 125829120 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚508931631513600 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚429 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚322 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0€ 1ãš2¾+- _wandbruntime‚4946  _runtime‚4946&ô#v¿+ð 1loss/FRt1/assignment_loss‚0.44636323387555876 1loss/FRt2/assignment_loss‚0.38827309513801794 /loss/FRt1/detection_loss‚0.5299096453627792 /loss/FRt2/detection_loss‚0.4375620951021698 epoch‚46 trainer/global_step‚18649 % _timestamp‚1.7866635974747438e+09 &loss/total_loss‚1.8021080694785256 _runtime‚ 4946.00455554 _step‚1537Åaóe{À+õ _runtime‚ 4946.00455554 _step‚1537 1loss/FRt1/assignment_loss‚0.44636323387555876 1loss/FRt2/assignment_loss‚0.38827309513801794 /loss/FRt1/detection_loss‚0.5299096453627792 /loss/FRt2/detection_loss‚0.4375620951021698 epoch‚46 trainer/global_step‚18649 % _timestamp‚1.7866635974747438e+09 &loss/total_loss‚1.8021080694785256 7rh–Á+‚(:‹ ±¥ùÓÕõ÷à gpu.0.gpu‚64.0 gpu.0.memory‚12, gpu.0.memoryAllocated‚7.0313262939453125) gpu.0.memoryAllocatedBytes‚ 6039863296 gpu.0.temp‚36.0 gpu.0.powerWatts‚193.292' gpu.0.enforcedPowerLimitWatts‚500.0 gpu.0.powerPercent‚38.6584 gpu.0.smClock‚1410 gpu.0.memoryClock‚1593$ gpu.0.correctedMemoryErrors‚1955# gpu.0.uncorrectedMemoryErrors‚0Q3aì‹Â+‚(:€ ±¥ùÓÿÓÒâ network.recv‚ 9.277722e+06 disk.dm-1.in‚ 45773.875# disk.dm-0.out‚63161.33935546875) proc.memory.availableMB‚ 5155.40234375# proc.memory.rssMB‚ 2338.62109375* proc.memory.percent‚14.273810386657715 proc.cpu.threads‚17 disk.dm-1.out‚ 91.421875 disk.dm-0.in‚70161.53515625 disk.sda3.out‚ 91.421875$ disk.nvme0c0n1.in‚70161.53515625( disk.nvme0c0n1.out‚63161.33935546875 disk.sda.in‚ 45773.875 disk.sda2.out‚0& disk.nvme0n1.out‚63161.33935546875 network.sent‚ 2.42485e+07# disk./.usageGB‚487.103328704834 disk.drbd2165.in‚0 disk.drbd2165.out‚0 disk.sda3.in‚ 45773.875 disk.sda1.out‚0 disk.sda2.in‚0$ memory_percent‚68.53392124176025) disk./.usagePercent‚32.68991309629685 disk.sda.out‚ 91.421875 disk.sda1.in‚0" disk.nvme0n1.in‚70161.53515625 cpu‚27.256406492275673ýÚá2Ã+- _wandbruntime‚4958  _runtime‚4958¦S'ÉÄ+à trainer/global_step‚18699 # _timestamp‚1.78666360948291e+09 _runtime‚4958.012900066 _step‚1538 ' lr-AdamW/pg1‚1.6247969993084893e-05 ' lr-AdamW/pg2‚1.6247969993084893e-05¸l·#ÎÅ+È ' lr-AdamW/pg1‚1.6247969993084893e-05 ' lr-AdamW/pg2‚1.6247969993084893e-05 trainer/global_step‚18699 # _timestamp‚1.78666360948291e+09 _runtime‚4958.012900066 _step‚1538‚ aCº>2Æ+- _wandbruntime‚4958  _runtime‚4958Œ`ÀcÇ+¿O _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 478113387b[,b'4688 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 4511297870698 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 122306526 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17104896 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚15688485 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚444766239490048 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚322 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 KDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚60 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 trainer/global_step‚18699 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 4781171621376 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 4520753292800 D;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚5264 KBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚4876 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 G>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4876 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚510221426065408 JDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚514742120308224 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚507205806096384 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 5494538240 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚ 131997968 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚209 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚511717162354462 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 125829120 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚510221426065408 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚ 132002257 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 125829120 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚262 IBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚60 F?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚60 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚69231197 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 4517085184 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 5368709120 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚62766773 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 122905600 C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚4289 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 122905600 IBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚58 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚449547352720896 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚20643840 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚511717103967082 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 125829120 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚510221408960512 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 4520711347712 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4289 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚69231195 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚322 % _timestamp‚1.7866636094838092e+09 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17104896 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 4520711347712 PCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚58387380 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 4511339218718 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚41348020 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 5368709120 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚514742179358208 PCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚59049984 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 5368709120 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚262 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚58791124 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚69231195 UHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚58390528 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 845178880 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚41945088 JBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚665 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚4287 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚58791066 JDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚514742120308224 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚510221408960512 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚444766218846208 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 831258624 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 BDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 4552982528 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚43102583 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚262 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚69231197 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 4654602240 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚62766773 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚41945088 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 M@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚59049984 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚62771060 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚449547411111424 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚15688541 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚322 _runtime‚4958.014049159 _step‚1539o‡PúÊ'È+ÄO WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17104896 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚15688485 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚444766239490048 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚322 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 KDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚60 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 trainer/global_step‚18699 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 4781171621376 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 4520753292800 D;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚5264 KBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚4876 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 G>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4876 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚510221426065408 JDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚514742120308224 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚507205806096384 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 5494538240 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚ 131997968 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚209 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚511717162354462 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 125829120 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚510221426065408 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚ 132002257 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 125829120 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚262 IBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚60 F?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚60 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚69231197 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 4517085184 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 5368709120 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚62766773 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 122905600 C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚4289 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 122905600 IBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚58 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚449547352720896 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚20643840 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚511717103967082 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 125829120 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚510221408960512 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 4520711347712 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4289 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚69231195 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚322 % _timestamp‚1.7866636094838092e+09 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17104896 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 4520711347712 PCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚58387380 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 4511339218718 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚41348020 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 5368709120 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚514742179358208 PCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚59049984 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 5368709120 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚262 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚58791124 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚69231195 UHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚58390528 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 845178880 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚41945088 JBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚665 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚4287 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚58791066 JDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚514742120308224 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚510221408960512 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚444766218846208 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 831258624 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 BDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 4552982528 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚43102583 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚262 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚69231197 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 4654602240 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚62766773 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚41945088 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 M@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚59049984 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚62771060 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚449547411111424 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚15688541 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚322 _runtime‚4958.014049159 _step‚1539 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 4781133874688 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 4511297870698 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 122306526ƒ „ï7…2É+- _wandbruntime‚4958  _runtime‚4958ûì7£Ï&Ê+ÉM T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚514768732767744 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 4552982528 B9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚5264 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚262 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚4295 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚41948160 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚58794145 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 122306526 G@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚60 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚ 132004808 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚69234748 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚43104788 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚511743578720658 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚262 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚62774353 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚444789286862848 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 4781424841216 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 125829120 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 125829120 D=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚60 _step‚1540 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚ 132009103 DDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59053056 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚511743520333230 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 5494538240 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 5494538240 G@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚70 EDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚322 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521206 trainer/global_step‚18699 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚41348068 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚322 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚20643840 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚58794075 NGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚68 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚61478912 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 5494538240 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 4654602240 B8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚4293 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚223 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 NADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚58387428 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚507231991889920 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚69234748 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 122905600 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚41937920 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚69234750 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚62770060 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654602240 EDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 132009103 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚4293 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚429 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 125829120 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚ 132004808 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 >5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5264 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚62774353 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚62770060 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚15689357 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚69234750 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 845178880 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 4781382903296 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚322 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 4552982528 H@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚665 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚444789266219008 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 4511569791378 NADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚59053056 SFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚62581760 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 5368709120 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 5368709120 % _timestamp‚1.7866636096445708e+09-ZU° Ë+ÎM SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 5368709120 % _timestamp‚1.7866636096445708e+09 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚514768732767744 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 4552982528 B9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚5264 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚262 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚4295 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚41948160 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚58794145 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 122306526 G@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚60 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚ 132004808 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚69234748 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚43104788 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚511743578720658 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚262 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚62774353 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚444789286862848 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 4781424841216 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 125829120 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 125829120 D=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚60 _step‚1540 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚ 132009103 DDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚š¥Ë$59053056 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚511743520333230 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 5494538240 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 5494538240 G@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚70 EDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚322 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521206 trainer/global_step‚18699 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚41348068 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚322 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚20643840 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚58794075 NGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚68 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚61478912 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 5494538240 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 4654602240 B8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚4293 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚223 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 NADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚58387428 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚507231991889920 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚69234748 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 122905600 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚41937920 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚69234750 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚62770060 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654602240 EDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 132009103 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚4293 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚429 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 125829120 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚ 132004808 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 >5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5264 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚62774353 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚62770060 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚15689357 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚69234750 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 845178880 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 4781382903296 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚322 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 4552982528 H@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚665 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚444789266219008 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 4511569791378 NADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚59053056 SFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚62581760 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 5368709120„ vœZ2Ì+- _wandbruntime‚4958  _runtime‚4958w{QvÍ+ð 0loss/FRt1/assignment_loss‚0.4113810941428556 1loss/FRt2/assignment_loss‚0.38072535339887315 /loss/FRt2/detection_loss‚0.4619410341231402 % _timestamp‚1.7866636096456096e+09 _runtime‚ 4958.17542313 0loss/FRt1/detection_loss‚0.45944580458375084 &loss/total_loss‚1.7134932862486196 epoch‚46 trainer/global_step‚18699 _step‚1541¥ÂK>{Î+õ 0loss/FRt1/detection_loss‚0.45944580458375084 &loss/total_loss‚1.7134932862486196 epoch‚46 trainer/global_step‚18699 _step‚1541 0loss/FRt1/assignment_loss‚0.4113810941428556 1loss/FRt2/assignment_loss‚0.38072535339887315 /loss/FRt2/detection_loss‚0.4619410341231402 % _timestamp‚1.7866636096456096e+09 _runtime‚ 4958.17542313… ;:²•Ï+‚(:Š À¥ùÓ³‘á gpu.0.gpu‚31.0 gpu.0.memory‚7, gpu.0.memoryAllocated‚7.0313262939453125) gpu.0.memoryAllocatedBytes‚ 6039863296 gpu.0.temp‚37.0 gpu.0.powerWatts‚226.296' gpu.0.enforcedPowerLimitWatts‚500.0 gpu.0.powerPercent‚45.2592 gpu.0.smClock‚1410 gpu.0.memoryClock‚1593$ gpu.0.correctedMemoryErrors‚1955# gpu.0.uncorrectedMemoryErrors‚0Lܯ‡Ð+‚(:ü À¥ùÓÃÏèâ) disk./.usagePercent‚32.68993229686191 disk.sda3.out‚ 91.64453125 disk.nvme0n1.in‚ 70166.5625& disk.nvme0n1.out‚63165.50927734375 disk.dm-1.in‚ 45773.875 disk.drbd2165.in‚0 disk.dm-0.in‚ 70166.5625( proc.memory.availableMB‚ 5155.3984375 network.sent‚ 2.4318178e+07$ disk./.usageGB‚487.1036148071289( disk.nvme0c0n1.out‚63165.50927734375 disk.sda.in‚ 45773.875 disk.sda.out‚ 91.64453125 disk.sda1.out‚0 disk.drbd2165.out‚0$ memory_percent‚68.53394508361816 network.recv‚ 9.304012e+06 disk.sda3.in‚ 45773.875 disk.nvme0c0n1.in‚ 70166.5625 disk.sda1.in‚0 disk.dm-1.out‚ 91.64453125# disk.dm-0.out‚63165.50927734375# proc.memory.rssMB‚ 2338.62109375 cpu‚27.255985514443296 disk.sda2.in‚0 disk.sda2.out‚0* proc.memory.percent‚14.273810386657715 proc.cpu.threads‚17˜cù2Ñ+- _wandbruntime‚4970  _runtime‚4970²CËÒ+Å ' lr-AdamW/pg1‚1.5027192860285449e-05 ' lr-AdamW/pg2‚1.5027192860285449e-05 trainer/global_step‚18749 % _timestamp‚1.7866636215110667e+09 _runtime‚4970.041108325 _step‚1542Úv¸yÐÓ+Ê % _timestamp‚1.7866636215110667e+09 _runtime‚4970.041108325 _step‚1542 ' lr-AdamW/pg1‚1.5027192860285449e-05 ' lr-AdamW/pg2‚1.5027192860285449e-05 trainer/global_step‚18749† ú6&2Ô+- _wandbruntime‚4970  _runtime‚4970»1®ÖÅ'Õ+¿O SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 4552982528 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚445918608130048 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚46135296 G>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4876 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚511537525850112 IBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚70 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 5494538240 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚322 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚41945088 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚41945088 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚262 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚58942231 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚15729468 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 122905600 JBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚665 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚61478912 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 4517085184 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚ 132345009 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚62936162 PCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚58387380 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚262 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 125829120 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚ 132340720 @7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚5264 M@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚59049984 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚450712060616704 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 5368709120 PCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚59049984 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 4532340058624 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 4532298113536 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚429 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 5494538240 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 4532340058624 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 4532298113536 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚69408845 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 845178880 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚4287 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17104896 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚516069883013632 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚322 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚511537542955008 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚511537542955008 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚508515095773184 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚262 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚69408847 _step‚1543 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚513038013841870 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 4552982528 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚58942301 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 4793452486656 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚508515112812544 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 122306526 F?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚60 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚209 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 4793406351360 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚41348020 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 5368709120 JDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 4654602240 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 831258624 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚322 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚322 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚450711993837568 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 125829120 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚516069883013632 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚20643840 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚62936162 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚516069823963648 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 122905600 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚4287 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚445918587486208 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 trainer/global_step‚18749 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 4628521206 KDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚60 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚516069823963648 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4289 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚15729400 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 4522859681306 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 125829120 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚223 % _timestamp‚1.7866636215119927e+09 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚43212831 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚513037955454490 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 5494538240 IBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚60 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚62931875 D;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚5264 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 BDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 4654602240 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 831258624 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚322 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚322 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚450711993837568 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 125829120 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚516069883013632 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚20643840 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚62936162 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚516069823963648 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 122905600 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚4287 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚445918587486208 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 trainer/global_step‚18749 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 4628521206 KDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚60 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚516069823963648 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4289 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚15729400 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 4522859681306 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 125829120 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚223 % _timestamp‚1.7866636215119927e+09 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚43212831 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚513037955454490 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 5494538240 IBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚60 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚62931875 D;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚5264 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 BDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4876 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚511537525850112 IBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚70 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 5494538240 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚322 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚41945088 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚41945088 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚262 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚58942231 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚15729468 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 122905600 JBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚665 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚61478912 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 4517085184 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚ 132345009 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚62936162 PCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚58387380 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚262 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 125829120 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚ 132340720 @7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚5264 M@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚59049984 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚450712060616704 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 5368709120 PCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚59049984 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 4532340058624 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 4532298113536 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚429 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 5494538240 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 4532340058624 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 4532298113536 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚69408845 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 845178880 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚4287 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17104896 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚516069883013632 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚322 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚511537542955008 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚511537542955008 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚508515095773184 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚262 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚69408847 _step‚1543 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚513038013841870 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 4552982528 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚58942301 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 4793452486656 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚508515112812544‡ p¯l2×+- _wandbruntime‚4970  _runtime‚4970O£G†{Ø+ÈM KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17104896 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚508541298606080 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚41348068 PDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59053056 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 4654602240 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 4532529182208 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚62935162 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚58945307 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚223 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚50326528 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 4517085184 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 122905600 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 4793649973248 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚ 132351855 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚516096436423168 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚58945245 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚450735284832256 YGDeviceStatsMonitor.on_train_batch_end/a¹1ÁSctive_bytes.small_pool.allocated‚ 4532571130368 G@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚62 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚511563848187904 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚69412400 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚262 D8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 5494538240 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 125829120 G@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚60 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚ 132347560 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚262 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚516096436423168 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 4552982528 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚61478912 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚513064430208066 trainer/global_step‚18749 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚62939455 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 4532529182208 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 125829120 NADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚59053056 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚41948160 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚4295 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚69412398 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚4293 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚20643840 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521206 IBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚60 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚62935162 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 B6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚322 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚262 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 4532571130368 H@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚665 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚450735355802624 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 5494538240 _runtime‚ 4970.20253425 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚69412400 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚41948160 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚508541281566720 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17104896 NGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚60 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 5368709120 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚322 EDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚429 >5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5264 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚511563865292800 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654602240 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚445941634859008 I@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚4876 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 4793700299776 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 4523131601986 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 5368709120 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚511563848187904 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚15730269 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚209 SFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚70970368 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 831258624 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 5494538240 NADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚58387428 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 122306526 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚322 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 132351855 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚43215038 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 % _timestamp‚1.7866636216722558e+091äÓ&Ù+ÍM EDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚429 >5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5264 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚511563865292800 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654602240 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚445941634859008 I@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚4876 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 4793700299776 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 4523131601986 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 5368709120 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚511563848187904 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚15730269 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚209 SFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚70970368 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 831258624 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 5494538240 NADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚58387428 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 122306526 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚322 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 132351855 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚43215038 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 % _timestamp‚1.7866636216722558e+09 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17104896 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚508541298606080 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚41348068 PDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59053056 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 4654602240 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 4532529182208 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚62935162 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚58945307 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚223 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚50326528 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 4517085184 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 122905600 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 4793649973248 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚ 132351855 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚516096436423168 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚58945245 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚450735284832256 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 4532571130368 G@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚62 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚511563848187904 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚69412400 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚262 D8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 5494538240 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 125829120 G@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚60 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚ 132347560 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚262 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚516096436423168 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 4552982528 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚61478912 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚513064430208066 trainer/global_step‚18749 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚62939455 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 4532529182208 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 125829120 NADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚59053056 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚41948160 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚4295 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚69412398 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚4293 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚20643840 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521206 IBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚60 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚62935162 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 B6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚322 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚262 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 4532571130368 H@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚665 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚450735355802624 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 5494538240 _runtime‚ 4970.20253425 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚69412400 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚41948160 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚508541281566720 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17104896 NGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚60 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 5368709120 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚322ˆ òyã2Ú+- _wandbruntime‚4970  _runtime‚4970ÐwtÛ+î /loss/FRt2/detection_loss‚0.4903434122678351 %loss/total_loss‚1.800660899603589 _runtime‚4970.203047491 /loss/FRt2/assignment_loss‚0.367387204335594 /loss/FRt1/detection_loss‚0.5008233501061982 epoch‚46 trainer/global_step‚18749 % _timestamp‚1.7866636216732476e+09 _step‚1545 1loss/FRt1/assignment_loss‚0.44210693289396197<~4TyÜ+ó /loss/FRt2/assignment_loss‚0.367387204335594 /loss/FRt1/detection_loss‚0.5008233501061982 epoch‚46 trainer/global_step‚18749 % _timestamp‚1.7866636216732476e+09 _step‚1545 1loss/FRt1/assignment_loss‚0.44210693289396197 /loss/FRt2/detection_loss‚0.4903434122678351 %loss/total_loss‚1.800660899603589 _runtime‚4970.203047491‰ P–ðÑ¡Ý+‚(:– Ï¥ùÓ‚†á gpu.0.gpu‚64.0 gpu.0.memory‚12, gpu.0.memoryAllocated‚7.0313262939453125) gpu.0.memoryAllocatedBytes‚ 6039863296 gpu.0.temp‚37.0 gpu.0.powerWatts‚217.704' gpu.0.enforcedPowerLimitWatts‚500.0) gpu.0.powerPercent‚43.540800000000004 gpu.0.smClock‚1410 gpu.0.memoryClock‚1593$ gpu.0.correctedMemoryErrors‚1955# gpu.0.uncorrectedMemoryErrors‚0Hú9ô†Þ+‚(:û Ï¥ùÓ¼—íã disk.sda1.in‚0) proc.memory.availableMB‚ 5155.30859375$ disk./.usageGB‚487.1209487915039 disk.sda1.out‚0 disk.sda2.out‚0# disk.nvme0n1.out‚63229.25390625 disk.drbd2165.out‚0# proc.memory.rssMB‚ 2338.62109375* proc.memory.percent‚14.273810386657715 proc.cpu.threads‚17 disk.sda3.in‚ 45773.875 disk.sda.out‚ 91.80859375 disk.sda2.in‚0! disk.nvme0n1.in‚ 70191.0234375 disk.dm-1.out‚ 91.80859375 disk.drbd2165.in‚0# memory_percent‚68.5344934463501 network.sent‚ 2.4387456e+07( disk./.usagePercent‚32.6910955950975 disk.dm-0.in‚ 70191.0234375 disk.dm-1.in‚ 45773.875 cpu‚27.255706823647156 network.recv‚ 9.330448e+06 disk.dm-0.out‚63229.25390625 disk.sda3.out‚ 91.80859375# disk.nvme0c0n1.in‚ 70191.0234375% disk.nvme0c0n1.out‚63229.25390625 disk.sda.in‚ 45773.875/d"2ß+- _wandbruntime‚4982  _runtime‚4982ÏPõ³Êà+Ä _step‚1546 ' lr-AdamW/pg1‚1.3853621657480136e-05 ' lr-AdamW/pg2‚1.3853621657480136e-05 trainer/global_step‚18799 $ _timestamp‚1.786663633742344e+09 _runtime‚4982.272340831¶²°Ïá+É trainer/global_step‚18799 $ _timestamp‚1.786663633742344e+09 _runtime‚4982.272340831 _step‚1546 ' lr-AdamW/pg1‚1.3853621657480136e-05 ' lr-AdamW/pg2‚1.3853621657480136e-05Š kòø.2â+- _wandbruntime‚4982  _runtime‚4982©“–+Å'ã+¿O YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚514358865329278 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 5368709120 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚69586495 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚512853659844608 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4289 UHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚66779136 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚514358806941898 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 5494538240 IBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚73 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚43323081 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚69586497 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚63096977 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚69586495 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚451876715331072 C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚4289 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚4287 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚15770312 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 4628521206 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 BDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 PCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚58387380 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚509824385449984 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚41945088 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚15770383 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 845178880 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚262 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚41945088 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚429 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚20643840 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 5368709120 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚262 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 _step‚1547 F>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 PCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚59049984 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚509824402489344 PIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚71 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚451876648551936 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚4287 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚59093466 KDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚60 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 JBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚665 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 4552982528 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚262 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17104896 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 4552982528 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 4543884879360 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 122306526 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚322 trainer/global_step‚18799 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚63101264 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚59093393 M@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚59049984 JDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚517397527619072 KBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚4876 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚447070976770048 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 125829120 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 JDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4876 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚209 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 4543926824448 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 4805692425728 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚512853642739712 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚322s Šwä+ÄO aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚447070956126208 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 5494538240 @7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚5264 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 122905600 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚63096977 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚46135296 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 4654602240 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 4534462839934 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 831258624 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 4805738561024 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚41348020 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 5494538240 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 5368709120 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17104896 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚512853642739712 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚69586497 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚322 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚43323083 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚223 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 4543884879360 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 D>DeviceStats9Rå9SMonitor.on_train_batch_start/oversize_segments.peak‚0 PCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚58387380 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚509824385449984 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚41945088 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚15770383 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 845178880 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚262 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚41945088 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚429 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚20643840 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 5368709120 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚262 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 _step‚1547 F>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 PCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚59049984 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚509824402489344 PIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚71 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚451876648551936 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚4287 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚59093466 KDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚60 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 JBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚665 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 4552982528 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚262 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17104896 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 4552982528 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 4543884879360 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 122306526 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚322 trainer/global_step‚18799 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚63101264 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚59093393 M@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚59049984 JDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚517397527619072 KBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚4876 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚447070976770048 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 125829120 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 JDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4876 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚209 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 4543926824448 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 4805692425728 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚512853642739712 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚322 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚514358865329278 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 5368709120 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚69586495 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚512853659844608 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4289 UHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚66779136 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚514358806941898 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 5494538240 IBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚73 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚43323081 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚69586497 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚63096977 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚69586495 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚451876715331072 C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚4289 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚4287 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚15770312 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 4628521206 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 B8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚322 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚517424140078592 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 4544157896192 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚512879965077504 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚262 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521206 IBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚60 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚512879982182400 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 4544157896192 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚509850588282880 >5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5264 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 122905600 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 122905600 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚447094003499008 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚514385281695474 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 125829120 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚322 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚517424140078592 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚63100264 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚451899943413248 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 4517085184 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚4293 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚43325288 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 4534652064526 E6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚322 D6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚322 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 4552982528 NADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚59053056 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 4544115948032 _step‚1548 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚63100264 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 4534693412594 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 5368709120 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 H@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚665 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚41948160 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚69590050 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚262 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚69590048 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654602240 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚15771125 K>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59053056 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 5494538240 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 _runtime‚4982.435565556 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚43325286 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚451900005995008 D=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚60 trainer/global_step‚18799 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 4654602240 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚41948160 B9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚5264 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚20643840 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚41348068 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚512879982182400 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚223 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 4805939914240 BDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 132694607 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 5368709120 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚63104557 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 5494538240 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚262 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚ 132690312 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚69590048 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 4805981852160 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚61478912 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17104896 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚59096411 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚15771182 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 845178880 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 125829120 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360¤ÉùÔ&ç+ÎM KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 _runtime‚4982.435565556 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚43325286 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚451900005995008 D=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚60 trainer/global_step‚18799 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 4654602240 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚41948160 B9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚5264 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚20643840 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚41348068 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚512879982182400 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚223 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 4805939914240 BDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 132694607 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 5368709120 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚63104557 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 5494538240 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚262 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚ 132690312 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚69590048 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 4805981852160 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚61478912 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17104896 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚59096411 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚15771182 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 845178880 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 125829120 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚63104557 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚512879965077504 E8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚322 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚517424140078592 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 4544157896192 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚512879965077504 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚262 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521206 IBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚60 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚512879982182400 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 4544157896192 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚509850588282880 >5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5264 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 122905600 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 122905600 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚447094003499008 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚514385281695474 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 125829120 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚322 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚517424140078592 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚63100264 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚451899943413248 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 4517085184 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚4293 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚43325288 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 4534652064526 E6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚322 D6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚322 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 4552982528 NADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚59053056 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 4544115948032 _step‚1548 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚63100264 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 4534693412594 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 5368709120 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 H@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚665 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚41948160 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚69590050 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚262 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚69590048 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654602240 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚15771125 K>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59053056 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 5494538240 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0Œ ÇÞJ2è+- _wandbruntime‚4982  _runtime‚4982›!d!ué+ï trainer/global_step‚18799 _runtime‚4982.435952107 _step‚1549 /loss/FRt2/detection_loss‚0.4768016606568734 &loss/total_loss‚1.8397287770032178 % _timestamp‚1.7866636339061446e+09 0loss/FRt1/assignment_loss‚0.5219256542371103 0loss/FRt2/assignment_loss‚0.3750692552709193 /loss/FRt1/detection_loss‚0.4659322068383148 epoch‚46’bÍYzê+ô epoch‚46 trainer/global_step‚18799 _runtime‚4982.435952107 _step‚1549 /loss/FRt2/detection_loss‚0.4768016606568734 &loss/total_loss‚1.8397287770032178 % _timestamp‚1.7866636339061446e+09 0loss/FRt1/assignment_loss‚0.5219256542371103 0loss/FRt2/assignment_loss‚0.3750692552709193 /loss/FRt1/detection_loss‚0.4659322068383148 ö”§[2ë+- _wandbruntime‚4987  _runtime‚4987"SB„áì+Û epoch‚46 3Purity/0FRt/event_proportion‚0.3301943838596344 -Purity/2FRt/FRt_purity‚0.5638312101364136 /particle/accuracy_2_of_2‚0.6817986369132996 (particle/accuracy‚0.6399855613708496 % _timestamp‚1.7866636392046208e+09 Purity/0FRt/event_purity‚1 particle/accuracy_1_of_1‚1 trainer/global_step‚18799 *loss/val_total_loss‚2.5414327633091607 .Purity/1FRt/FRt_purity‚0.46882185339927673 _runtime‚4987.734813378 *validation_accuracy‚0.2101372480392456 2Purity/*FRt/event_proportion‚0.669805645942688 3Purity/1FRt/event_proportion‚0.6297432780265808 +jet/accuracy_1_of_1‚0.31040075421333313 +particle/sensitivity‚0.6296515464782715 'particle/f_score‚0.5538610219955444 0Purity/*FRt/event_purity‚0.46644219756126404 0Purity/1FRt/event_purity‚0.46882185339927673 /Purity/2FRt/event_purity‚0.4274119734764099 *jet/accuracy_2_of_2‚0.2101372480392456 6validation_average_jet_accuracy‚0.3152996301651001 *jet/accuracy_1_of_2‚0.5734885931015015 .Purity/*FRt/FRt_purity‚0.47967442870140076 2Purity/2FRt/event_proportion‚0.040062315762043 /particle/accuracy_1_of_2‚0.9230902791023254 +particle/specificity‚0.6456539034843445 _step‚1550‘9æí+à _step‚1550 epoch‚46 3Purity/0FRt/event_proportion‚0.3301943838596344 -Purity/2FRt/FRt_purity‚0.5638312101364136 /particle/accuracy_2_of_2‚0.6817986369132996 (particle/accuracy‚0.6399855613708496 % _timestamp‚1.7866636392046208e+09 Purity/0FRt/event_purity‚1 particle/accuracy_1_of_1‚1 trainer/global_step‚18799 *loss/val_total_loss‚2.5414327633091607 .Purity/1FRt/FRt_purity‚0.46882185339927673 _runtime‚4987.734813378 *validation_accuracy‚0.2101372480392456 2Purity/*FRt/event_proportion‚0.669805645942688 3Purity/1FRt/event_proportion‚0.6297432780265808 +jet/accuracy_1_of_1‚0.31040075421333313 +particle/sensitivity‚0.6296515464782715 'particle/f_score‚0.5538610219955444 0Purity/*FRt/event_purity‚0.46644219756126404 0Purity/1FRt/event_purity‚0.46882185339927673 /Purity/2FRt/event_purity‚0.4274119734764099 *jet/accuracy_2_of_2‚0.2101372480392456 6validation_average_jet_accuracy‚0.3152996301651001 *jet/accuracy_1_of_2‚0.5734885931015015 .Purity/*FRt/FRt_purity‚0.47967442870140076 2Purity/2FRt/event_proportion‚0.040062315762043 /particle/accuracy_1_of_2‚0.9230902791023254 +particle/specificity‚0.6456539034843445Ž ž§×N–î+‚(:‹ Þ¥ùÓ‹˜¢á gpu.0.gpu‚64.0 gpu.0.memory‚12, gpu.0.memoryAllocated‚7.0313262939453125) gpu.0.memoryAllocatedBytes‚ 6039863296 gpu.0.temp‚36.0 gpu.0.powerWatts‚123.223' gpu.0.enforcedPowerLimitWatts‚500.0 gpu.0.powerPercent‚24.6446 gpu.0.smClock‚1410 gpu.0.memoryClock‚1593$ gpu.0.correctedMemoryErrors‚1955# gpu.0.uncorrectedMemoryErrors‚0vÆ$ï+‚(:‚ Þ¥ùÓìÜÐã% disk./.usageGB‚487.17608642578125 disk.sda2.in‚0 disk.sda2.out‚0! disk.nvme0n1.in‚ 70261.5546875 disk.dm-1.out‚ 92.26171875 disk.drbd2165.in‚0 disk.sda1.in‚0 disk.dm-0.in‚ 70261.5546875$ memory_percent‚68.58229637145996( proc.memory.availableMB‚ 5147.4765625# proc.memory.rssMB‚ 2346.43359375 proc.cpu.threads‚17) disk./.usagePercent‚32.69479592799741 disk.sda3.out‚ 92.26171875 disk.sda.in‚ 45773.875& disk.nvme0n1.out‚63291.43798828125* proc.memory.percent‚14.321494102478027 cpu‚27.26071007348538 disk.sda3.in‚ 45773.875# disk.nvme0c0n1.in‚ 70261.5546875( disk.nvme0c0n1.out‚63291.43798828125 disk.sda.out‚ 92.26171875 disk.sda1.out‚0 disk.dm-1.in‚ 45773.875 disk.drbd2165.out‚0# disk.dm-0.out‚63291.43798828125 network.sent‚ 2.446004e+07 network.recv‚ 9.3596e+06Q82ð+- _wandbruntime‚5000  _runtime‚5000Õy9-ñ+Å ÁPWöÄ% _timestamp‚1.7866636523138285e+09 _runtime‚5000.843863847 _step‚1551 ' lr-AdamW/pg1‚1.2727331760954002e-05 ' lr-AdamW/pg2‚1.2727331760954002e-05 trainer/global_step‚18849¥!HÐò+Ê trainer/global_step‚18849 % _timestamp‚1.7866636523138285e+09 _runtime‚5000.843863847 _step‚1551 ' lr-AdamW/pg1‚1.2727331760954002e-05 ' lr-AdamW/pg2‚1.2727331760954002e-05 %ar2ó+- _wandbruntime‚5000  _runtime‚5000¢4.Å'ô+¿O \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚514560696844288 JDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚519120236213248 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚209 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 4550090622643 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 125829120 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚262 trainer/global_step‚18849 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚41945088 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚69824753 PIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚68 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 4822108231168 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 5494538240 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 5368709120 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 4559581314048 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 831258624 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚61478912 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 4628521206 F>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚41945088 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚4287 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚43473947 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚69824751 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 4517085184 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 5368709120 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 125829120 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚223 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 _step‚1552 D;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚5264 C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚4289 @7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚5264 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚63284266 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 122905600 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 845178880 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 M@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚59049984 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚514560713949184 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚43473945 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚322 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚511514591657984 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 % _timestamp‚1.7866636523148584e+09 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚429 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚519120295263232 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 122905600 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚448522595074048 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 4550049274623 F?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚60 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 4559539368960 JDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4876 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚519120295263232 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚519120236213248 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 122306526 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17104896 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 4822064193024 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 125829120 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚262 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚63288553 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚511514574618624 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚41348020 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 5494538240 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚ 133113306 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 5494538240 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚322 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 KDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚60 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 4559539368960 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 4654602240 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚ 133109017 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚63288553 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚4287 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚516064623893247 IBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚60 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚514560696844288 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚15819944 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 4552982528 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚59293891 IBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚70 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚59293821 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚453344723949056 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚516064682280627 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 4559581314048 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚15819876 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚69824751 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 4654602240 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚514560713949184 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4289 UHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚64681984 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 PCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚59049984 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17104896 _runtime‚5000.845112207»Á°JÊ'õ+ÄO GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 KDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚60 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 4559539368960 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 4654602240 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚ 133109017 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚63288553 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚4287 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚516064623893247 IBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚60 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚514560696844288 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚15819944 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 4552982528 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚59293891 IBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚70 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚59293821 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚453344723949056 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚516064682280627 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 4559581314048 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚15819876 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚69824751 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 4654602240 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚514560713949184 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4289 UHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚64681984 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 PCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚59049984 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17104896 _runtime‚5000.845112207 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚514560696844288 JDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚519120236213248 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚209 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 4550090622643 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 125829120 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚262 trainer/global_step‚18849 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚41945088 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚69824753 PIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚68 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 4822108231168 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 5494538240 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 5368709120 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 4559581314048 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 831258624 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚61478912 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 4628521206 F>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚41945088 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚4287 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚43473947 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚69824751 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 4517085184 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 5368709120 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 125829120 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚223 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 _step‚1552 D;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚5264 C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚4289 @7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚5264 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚63284266 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 122905600 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 845178880 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 M@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚59049984 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚514560713949184 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚43473945 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚322 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚511514591657984 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 % _timestamp‚1.7866636523148584e+09 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚429 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚519120295263232 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 122905600 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚448522595074048 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 4550049274623 F?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚60 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 4559539368960 JDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4876 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚519120295263232 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚519120236213248 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 122306526 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17104896 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 4822064193024 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 125829120 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚262 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚63288553 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚511514574618624 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚41348020 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 5494538240 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚ 133113306 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 5494538240 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚322 ióJ¿2ö+- _wandbruntime‚5001  _runtime‚50013±%÷Ï&÷+ÉM T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚519146848672768 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17104896 H@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚665 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚63287553 PDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654602240 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚516091040259395 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 125829120 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚41948160 NGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚66 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚453368016153088 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚41348068 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚4293 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚514587036286976 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚43476150 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 4822353062400 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚ 133120152 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚61478912 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚516091098646823 E6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚322 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 NADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚58387428 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚511540760412160 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 122306526 trainer/global_step‚18849 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 5368709120 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚43476152 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚262 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚519146789619712 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 4559812385792 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521206 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 4517085184 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚322 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 4559770437632 SFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚64678912 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚453367951474176 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 4559812385792 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚41948160 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 4550321195303 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚322 _step‚1553 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 122905600 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚448545642446848 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚322 % _timestamp‚1.7866636524768395e+09 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 831258624 IBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚60 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚69828304 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚448545663090688 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 _runtime‚5001.007122418 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 DDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 5494538240 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 4559770437632 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚15820709 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚4295 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 4552982528 B9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚5264 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚59296859 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚15820775 EDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 133120152 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚69828304 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚223 D=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚60 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚514587036286976 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚44035072 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 5368709120 K>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59053056 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚63287553 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚514587019182080 I@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚4876 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 845178880 B5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5264 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0=ÝøÛrø+ÎM H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚ 133115857 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚209 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 5494538240 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚262 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 5494538240 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 4559770437632 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚15820709 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚4295 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 4552982528 B9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚5264 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚59296859 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚15820775 EDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 133120152 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚69828304 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚223 D=DeviceStatsMonitor.on_train_batch_end/segms£îÂbent.small_pool.peak‚60 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚514587036286976 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚44035072 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 5368709120 K>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59053056 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚63287553 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚514587019182080 I@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚4876 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 845178880 B5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5264 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚519146848672768 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17104896 H@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚665 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚63287553 PDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654602240 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚516091040259395 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 125829120 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚41948160 NGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚66 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚453368016153088 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚41348068 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚4293 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚514587036286976 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚43476150 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 4822353062400 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚ 133120152 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚61478912 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚516091098646823 E6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚322 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 NADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚58387428 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚511540760412160 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 122306526 trainer/global_step‚18849 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 5368709120 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚43476152 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚262 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚519146789619712 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 4559812385792 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521206 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 4517085184 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚322 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 4559770437632 SFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚64678912 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚453367951474176 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 4559812385792 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚41948160 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 4550321195303 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚322 _step‚1553 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 122905600 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚448545642446848 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚322 % _timestamp‚1.7866636524768395e+09 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 831258624 IBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚60 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚69828304 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚448545663090688 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 _runtime‚5001.007122418 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 DDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 D;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚5264 IBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚66 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 4628521206 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚262 JDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚209 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 5494538240 trainer/global_step‚18899 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚515876830838784 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚322 PCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚59049984 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚41945088 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 4834388514816 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚512823881334784 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚43584197 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 5368709120 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚429 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚43584195 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚449674963714048 IBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚60 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚20643840 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 G>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4876 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 122905600 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚517385533768035 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚ 133451769 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 4571126134784 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4289 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 5494538240 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 5368709120 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚520447939868672 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚41945088 KDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚60 KBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚4876 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚322 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚4287 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 4571168079872 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚59444635 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 4561611085231 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 JDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚209 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 5494538240 trainer/global_step‚18899 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚515876830838784 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚322 PCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚59049984 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚41945088 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 4834388514816 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚512823881334784 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚43584197 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 5368709120 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚429 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚43584195 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚449674963714048 IBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚60 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚20643840 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 G>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4876 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 122905600 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚517385533768035 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚ 133451769 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 4571126134784 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4289 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 5494538240 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 5368709120 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚520447939868672 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚41945088 KDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚60 KBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚4876 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚322 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚4287 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 4571168079872 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚59444635 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 4561611085231 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 JDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 D;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚5264 IBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚66 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 4628521206 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚262 JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654602240 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚43586400 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 5368709120 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 125829120 >5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5264 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 4552982528 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚70005954 NGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚75 E?DeviceStatsMoâ?–ìŸ nitor.on_train_batch_end/oversize_allocations.peak‚0 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚70005954 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚63452655 B9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚5264 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 4834633870336 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚517411891746803 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚520474552328192 DDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521206 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚41948160 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 5368709120 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚ 133462904 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚70005956 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚59447684 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚46132224 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 4834587738112 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 5494538240 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚59447607 SFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚66776064 E8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 4561841657843 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚322 _step‚1557 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17104896 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚41948160 H@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚665 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚63456948 G@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚77 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚41348068 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚223 NADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚59053056 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚4293 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚512850050088960 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚70005956 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚63452655 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚209 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 4571357203456 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 125829120 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚322 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚520474493275136 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 5494538240 PDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚4293 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚449698031730688 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚61478912 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 K>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59053056 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 4571399151616 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚429 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚454532598824960 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚322 NADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚58387428 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚262 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚4295 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 5494538240 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 122306526 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 133462904 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 G@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚60 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17104896 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚454532665601024 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 % _timestamp‚1.7866636649648607e+09 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚ 133458609 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 4571399151616 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 4571357203456 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 845178880 _runtime‚5013.495363125 IBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚60È3äÔ&†,ÎM D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚223 NADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚59053056 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚4293 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚512850050088960 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚70005956 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚63452655 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚209 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 4571357203456 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 125829120 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚322 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚520474493275136 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 5494538240 PDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚4293 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚449698031730688 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚61478912 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 K>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59053056 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 4571399151616 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚429 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚454532598824960 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚322 NADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚58387428 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚262 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚4295 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 5494538240 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 122306526 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 133462904 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 G@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚60 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17104896 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚454532665601024 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 % _timestamp‚1.7866636649648607e+09 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚ 133458609 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 4571399151616 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 4571357203456 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 845178880 _runtime‚5013.495363125 IBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚60 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚515903136071680 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚43586402 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 4517085184 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚517411950134231 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 4561883005911 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 4654602240 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚515903153176576 EDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654602240 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚43586400 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 5368709120 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 125829120 >5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5264 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 4552982528 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚70005954 NGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚75 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚70005954 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚63452655 B9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚5264 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 4834633870336 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚517411891746803 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚520474552328192 DDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521206 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚41948160 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 5368709120 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚ 133462904 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚70005956 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚59447684 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚46132224 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 4834587738112 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 5494538240 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚59447607 SFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚66776064 E8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 4561841657843 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚322 _step‚1557 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17104896 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚41948160 H@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚665 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚63456948 G@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚77 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚41348068• àæÛ_2‡,- _wandbruntime‚5013  _runtime‚5013ËqD\vˆ,ð 0loss/FRt1/assignment_loss‚0.5505450882376589 1loss/FRt2/assignment_loss‚0.36318555765442356 0loss/FRt1/detection_loss‚0.47915498942865536 /loss/FRt2/detection_loss‚0.5003155593527868 $ _timestamp‚1.786663664965955e+09 &loss/total_loss‚1.8932011946735248 epoch‚47 trainer/global_step‚18899 _runtime‚5013.495874672 _step‚1558Z0Â{‰,õ 0loss/FRt1/assignment_loss‚0.5505450882376589 1loss/FRt2/assignment_loss‚0.36318555765442356 0loss/FRt1/detection_loss‚0.47915498942865536 /loss/FRt2/detection_loss‚0.5003155593527868 $ _timestamp‚1.786663664965955e+09 &loss/total_loss‚1.8932011946735248 epoch‚47 trainer/global_step‚18899 _runtime‚5013.495874672 _step‚1558– ‘€ƒ2–Š,‚(:‹ ü¥ùÓô·µá gpu.0.gpu‚64.0 gpu.0.memory‚12, gpu.0.memoryAllocated‚7.0313262939453125) gpu.0.memoryAllocatedBytes‚ 6039863296 gpu.0.temp‚37.0 gpu.0.powerWatts‚225.242' gpu.0.enforcedPowerLimitWatts‚500.0 gpu.0.powerPercent‚45.0484 gpu.0.smClock‚1410 gpu.0.memoryClock‚1593$ gpu.0.correctedMemoryErrors‚1955# gpu.0.uncorrectedMemoryErrors‚0°Ù÷™‹,‚(:Ž ü¥ùÓƒøîæ& disk.nvme0n1.out‚63303.42236328125 disk.dm-1.in‚45797.87890625 disk.dm-1.out‚92.90625 proc.cpu.threads‚17 network.sent‚ 2.4597134e+07 disk.sda3.out‚92.90625$ disk.nvme0c0n1.in‚76197.32421875 disk.sda1.in‚0 disk.dm-0.in‚76197.32421875* proc.memory.percent‚14.321494102478027 disk.sda.in‚45797.87890625 disk.sda2.in‚0 disk.drbd2165.in‚0# disk.dm-0.out‚63303.42236328125$ memory_percent‚68.58129501342773' proc.memory.availableMB‚ 5147.640625 disk.sda2.out‚0 network.recv‚ 9.409736e+06) disk./.usagePercent‚32.69479234389193 disk.sda3.in‚45797.87890625 disk.sda1.out‚0 disk.drbd2165.out‚0# proc.memory.rssMB‚ 2346.43359375 cpu‚27.259672340089196% disk./.usageGB‚487.17603302001953( disk.nvme0c0n1.out‚63303.42236328125 disk.sda.out‚92.90625" disk.nvme0n1.in‚76197.32421875: v¿2Œ,- _wandbruntime‚5026  _runtime‚5026úÄ<Ê,Ä $ _timestamp‚1.786663677710764e+09 _runtime‚5026.240953374 _step‚1559 ' lr-AdamW/pg1‚1.0616882203284828e-05 ' lr-AdamW/pg2‚1.0616882203284828e-05 trainer/global_step‚18949|FÇÏŽ,É ' lr-AdamW/pg1‚1.0616882203284828e-05 ' lr-AdamW/pg2‚1.0616882203284828e-05 trainer/global_step‚18949 $ _timestamp‚1.786663677710764e+09 _runtime‚5026.240953374 _step‚1559— N~ïõ2,- _wandbruntime‚5026  _runtime‚5026OeÅmÅ',¿O VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚521775702574080 IBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚75 PIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚73 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 4846690855424 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 5494538240 _runtime‚5026.245991414 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 4654602240 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17104896 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚517192930623488 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚223 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚61478912 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 125829120 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚41945088 KBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚4876 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚15900992 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚322 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚4287 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚455673974977024 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 PCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚58387380 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚322 % _timestamp‚1.7866636777118633e+09 _step‚1560 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚518706385255443 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚70180053 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚262 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚63618757 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚59595364 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚43694445 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 4582754845696 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚63618757 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 5494538240 JDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 BDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 4573172895839 KDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚60 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚521775643524096 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17104896 @7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚5264 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚517192947728384 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚517192930623488 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 4552982528 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 122905600 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚43694447 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 4517085184 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 122306526 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 4628521206 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 5368709120 G>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4876 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚517192947728384 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚429 UHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚68876288 C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚4289 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 5368709120 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚41945088 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚59595439 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 831258624 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚48232448 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚262 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚63614470 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 4582712900608 D;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚5264 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚63614470 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 5368709120 trainer/global_step‚18949 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚20643840 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚455674043853312 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚514133153972224 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 4582754845696 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚70180051 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚262 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4289 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 4552982528 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚521775702574080 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚70180053 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 125829120 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚ 133794521ñ%l‘,ÄO O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 4654602240 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17104896 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚517192930623488 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚223 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚61478912 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 125829120 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚41945088 KBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚4876 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚15900992 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚322 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚4287 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚455673974977024 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 PCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚58387380 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚322 % _timestamp‚1.7866636777118633e+09 Þ¬!_step‚1560 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚518706385255443 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚70180053 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚262 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚63618757 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚59595364 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚43694445 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 4582754845696 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚63618757 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 5494538240 JDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 BDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 4573172895839 KDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚60 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚521775643524096 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17104896 @7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚5264 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚517192947728384 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚517192930623488 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 4552982528 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 122905600 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚43694447 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 4517085184 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 122306526 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 4628521206 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 5368709120 G>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4876 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚517192947728384 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚429 UHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚68876288 C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚4289 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 5368709120 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚41945088 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚59595439 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 831258624 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚48232448 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚262 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚63614470 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 4582712900608 D;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚5264 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚63614470 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 5368709120 trainer/global_step‚18949 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚20643840 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚455674043853312 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚514133153972224 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 4582754845696 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚70180051 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚262 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4289 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 4552982528 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚521775702574080 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚70180053 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 125829120 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚ 133794521 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚521775702574080 IBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚75 PIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚73 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 4846690855424 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 5494538240 _runtime‚5026.245991414 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0˜ ±V2’,- _wandbruntime‚5026  _runtime‚5026Ãp´JÏ&“,ÉM WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 125829120 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚521802255983616 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚521802196930560 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 133805656 B9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚5264 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 4846937013760 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 5368709120 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 4654602240 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 4582943969280 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚455697337384448 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚44035072 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚61478912 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚262 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚59598435 IBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚60 % _timestamp‚1.7866636778734984e+09 K>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59053056 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521206 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 >5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5264 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚70183604 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚223 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚15901783 NADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚58387428 PDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654602240 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚70183606 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚70183604 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚ 133801361 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚63617757 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚429 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚209 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚41948160 G@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚69 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚322 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚517219270066176 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 122905600 NADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚59053056 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 4582985917440 trainer/global_step‚18949 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17104896 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 4552982528 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚450850400370688 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚ 133805656 E6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚322 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 125829120 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 125829120 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚59598366 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚43696650 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 122905600 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚20643840 B8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚514159339765760 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚262 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚63617757 _runtime‚5026.403902265 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚517219252961280 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 5494538240#_H"Ô&”,ÎM REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚70183606 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 845178880 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚41948160 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 4517085184 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 4573444816519 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 5494538240 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚322 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 125829120 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 125829120 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚59598366 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚43696650 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 122905600 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚20643840 B8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚514159339765760 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚262 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚63617757 _runtime‚5026.403902265 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚517219252961280 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 5494538240 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 125829120 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚521802255983616 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚521802196930560 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 133805656 B9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚5264 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 4846937013760 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 5368709120 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 4654602240 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 4582943969280 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚455697337384448 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚44035072 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚61478912 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚262 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚59598435 IBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚60 % _timestamp‚1.7866636778734984e+09 K>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59053056 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521206 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 >5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5264 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚70183604 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚223 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚15901783 NADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚58387428 PDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654602240 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚70183606 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚70183604 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚ 133801361 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚63617757 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚429 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚209 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚41948160 G@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚69 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚322 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚517219270066176 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 122905600 NADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚59053056 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 4582985917440 trainer/global_step‚18949 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17104896 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 4552982528 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚450850400370688 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚ 133805656 EDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚523103347179520 D;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚5264 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚20643840 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 G>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4876 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4289 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 4584734706447 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚59746052 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 4584776054467 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 5494538240 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚262 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚518509064617984 IBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚78 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚223 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚48232448 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 4858928706048 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚ 134141562 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚456838698576384 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 125829120 KDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚60 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚523103406229504 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚322 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 5368709120 C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚4289 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 4594341611520 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚63779572 KBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚4876 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚520027236742851 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚41348020 PIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚76 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 125829120 F>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚63783859 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚41945088 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 4594299666432 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚322 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 4552982528 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚4287 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 % _timestamp‚1.7866636898966362e+09ÑÊÊ',ÄO ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 125829120 F>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚63783859 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚41945088 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 4594299666432 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚322 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 4552982528 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚4287 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 % _timestamp‚1.7866636898966362e+09 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚ 134137273 JBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚665 PCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚59049984 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚15941279 UHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚68876288 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 125829120 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 F?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚60 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚209 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 _step‚1564 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 4594341611520 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚41945088 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚518509047513088 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 4552982528 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚451979721637888 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚322 _runtime‚5038.427983898 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚63779572 M@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚59049984 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚518509064617984 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚518509047513088 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚43804697 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚15941355 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 5494538240 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 5368709120 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚523103406229504 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17104896 JDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚523103347179520 D;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚5264 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚20643840 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 G>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4876 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4289 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 4584734706447 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚59746052 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 4584776054467 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 5494538240 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚262 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚518509064617984 IBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚78 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚223 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚48232448 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 4858928706048 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚ 134141562 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚456838698576384 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 125829120 KDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚60 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚523103406229504 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚322 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 5368709120 C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚4289 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 4594341611520 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚63779572 KBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚4876 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚520027236742851 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚41348020 PIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚76œ ŸY‰š2ž,- _wandbruntime‚5038  _runtime‚50385cÎ&Ÿ,ÈM XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 5368709120 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 845178880 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚20643840 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 5494538240 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 4517085184 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 5494538240 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚515468646481920 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚322 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚429 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚41948160 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 134148408 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 G@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚60 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 4594530735104 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚523129959639040 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654602240 H@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚665 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 122306526 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 4594572683264 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚4293 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521206 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚515468629442560 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚63782859 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚59748975 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚209 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚ 134144113 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚456861919195648 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 K>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59053056 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚523129900585984 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17104896 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚43806902 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 125829120 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚262 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚ 134148408 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 4654602240 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚518535369850880 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 4859223252480 D6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚322 IBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚60 PDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚61478912 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 5368709120 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚4295 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17104896 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 122905600 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚59749052 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚15942075 SFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚73067520 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚518535386955776 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚322 D=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚60 >5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5264 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚518535369850880 NADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚58387428 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 4552982528 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 831258624 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚262 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚70361254 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚452002769010688 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 4859170828800 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚520053594721619 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚322 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚43806900 NGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚75 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚520053653109047 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚63787152 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 BDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚262 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 122905600 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚63782859 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚52423680 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 125829120 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 trainer/global_step‚18999 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚4293 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 4585006627127 I@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚4876 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚70361254 B9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚5264 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚70361256 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚41948160 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚456861992263168 % _timestamp‚1.7866636900609365e+09 _step‚1565 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚223 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 5494538240V1‚] ,ÍM E6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚322 IBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚60 PDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚61478912 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 5368709120 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚4295 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17104896 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 122905600 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚59749052 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚15942075 SFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚73067520 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚518535386955776 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚322 D=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚60 >5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5264 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚518535369850880 NADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚58387428 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 4552982528 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 831258624 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚262 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚70361254 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚452002769010688 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 4859170828800 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚520053594721619 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚322 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚43806900 NGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚75 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚520053653109047 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚63787152 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 BDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚262 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 122905600 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚63782859 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚52423680 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 125829120 D>DeviceStatsMonitor.on_train_batch_end/segment.large_poo–»ùvl.freed‚0 trainer/global_step‚18999 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚4293 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 4585006627127 I@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚4876 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚70361254 B9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚5264 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚70361256 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚41948160 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚456861992263168 % _timestamp‚1.7866636900609365e+09 _step‚1565 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚223 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 5494538240 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 5368709120 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 845178880 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚20643840 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 5494538240 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 4517085184 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 5494538240 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚515468646481920 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚322 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚429 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚41948160 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 134148408 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 G@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚60 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 4594530735104 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚523129959639040 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654602240 H@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚665 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 122306526 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 4594572683264 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚4293 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521206 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚515468629442560 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚63782859 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚59748975 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚209 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚ 134144113 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚456861919195648 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 K>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59053056 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚523129900585984 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17104896 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚43806902 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 125829120 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚262 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚ 134148408 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 4654602240 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚518535369850880 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 4859223252480 DDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚521348029842879 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 125829120 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚322 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚ 134480025 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚63948961 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 122905600 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 845178880 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚44038144 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 4605928377344 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 4517085184 F?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚60 _step‚1568 C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚4289 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚63948961 G>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4876 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚524431050834944 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4289 % _timestamp‚1.7866637021675458e+09 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17104896 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚15981966 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚20643840 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 5368709120 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚70535351 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 4654602240 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚519825164402688 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚524431050834944 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚41945088 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚70535351 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚63944674 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 4605928377344 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚4287 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 5494538240 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚63944674 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚59896913 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚43914945 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚15981900 PCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚58387380 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚453132069634048 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 5368709120 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚262 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚524431109884928 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚70535353 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚516751733325824 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚524431109884928 JDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 122905600U“—Ê'«,ÄO XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚15981966 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚20643840 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 5368709120 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚70535351 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 4654602240 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚519825164402688 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚524431050834944 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚41945088 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚70535351 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚63944674 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 4605928377344 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚4287 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 5494538240 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚63944674 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚59896913 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚43914945 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚15981900 PCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚58387380 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚453132069634048 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 5368709120 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚262 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚524431109884928 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚70535353 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚516751733325824 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚524431109884928 JDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 122905600 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚429 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 5494538240 IBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚60 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 @7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚5264 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚453132090277888 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 4628521206 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 4596337865075 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚322 KDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚60 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17104896 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 125829120 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚262 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚70535353 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚59896845 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 trainer/global_step‚19049 KBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚4876 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚209 BDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚521348029842879 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 125829120 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚322 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚ 134480025 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚63948961 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 122905600 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 845178880 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚44038144 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 4605928377344 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 4517085184 F?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚60 _step‚1568 C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚4289 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚63948961 G>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4876 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚524431050834944 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4289 % _timestamp‚1.7866637021675458e+09 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17104896  þŸž2¬,- _wandbruntime‚5050  _runtime‚5050I½fY­,ÉM H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚ 134486865 G@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚60 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚70538904 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚61478912 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 5368709120 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚63947961 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 4606159449088 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17104896 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚41348068 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 5368709120 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚70538906 E6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚322 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 >5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5264 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚59899836 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚429 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 831258624 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚322 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 4606159449088 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚63947961 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 4871519129600 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 4606117500928 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚519851486740480 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 5494538240 D=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚60 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚4295 G@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚68 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚44035072 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 DDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654602240 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 4552982528 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 4596568437735 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 122905600 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚41948160 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚458026592101376 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.aðG¿ll.freed‚521374446209027 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚524457663294464 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚519851486740480 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚524457604241408 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚63952254 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 5494538240 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚15982686 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚20643840 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚4293 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 4552982528 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚41948160 E8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚521374504596455 PDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59053056 I@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚4876 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚59899904 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚516777919119360 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 134491160 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 845178880 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521206 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 4517085184 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 4654602240 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚519851503845376 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚70538904ƒ{Í0Ô&®,ÎM QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 5494538240 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 NADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚58387428 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚322 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 >5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5264 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚59899836 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚429 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 831258624 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚322 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 4606159449088 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚63947961 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 4871519129600 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 4606117500928 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚519851486740480 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 5494538240 D=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚60 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚4295 G@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚68 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚44035072 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 DDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654602240 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 4552982528 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 4596568437735 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 122905600 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚41948160 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚458026592101376 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚521374446209027 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚524457663294464 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚519851486740480 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚524457604241408 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚63952254 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 5494538240 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚15982686 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚20643840 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚4293 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 4552982528 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚41948160 E8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚521374504596455 PDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59053056 I@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚4876 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚59899904 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚516777919119360 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 134491160 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 845178880 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521206 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 4517085184 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 4654602240 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚519851503845376 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚70538904 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚ 134486865 G@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚60 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚70538904 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚61478912 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 5368709120 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚63947961 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 4606159449088 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17104896 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚41348068 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 5368709120 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚70538906 EDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚64109776 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚518061023002624 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 5494538240 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 5494538240 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 125829120 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚322 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚4287 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚525758754490368 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 4617473198080 BDeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4289 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 845178880 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 831258624 PCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚58387380 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚16022510 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚ 134822777 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚41945088 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚61478912 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚41348020 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 $ _timestamp‚1.786663714699045e+09 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚64114063 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚223 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚525758754490368 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚262 @7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚5264 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚522668881330287 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 5368709120 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 F?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚60 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 5368709120 trainer/global_step‚19099 _runtime‚5063.232948269 KDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚60 JDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4876 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 4654602240 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 4552982528 KBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚4876 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 4607899675683 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 4607858327663 M@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚59049984 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17104896 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚44025195 UHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚70973440 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 122306526 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 5494538240 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚521141281292288 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 JBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚665 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚459168012475904 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚50329600 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 125829120 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 122905600 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚4287 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚322 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚64109776 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚525758813540352 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚521141281292288 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 4552982528 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚70713001 IBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚80 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0.–Rÿ¹,ÃO JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚262 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚525758813540352 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚ 134827066 D;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚5264 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 4628521206 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚518061040041984 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 4617473198080 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 122905600 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚70713003 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 125829120 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 5368709120 PIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚78 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚459167941502464 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚20643840 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚521141298397184 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚521141298397184 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 4883503228416 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚262 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚16022588 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚454284458917888 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 F>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚64109776 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚518061023002624 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 5494538240 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 5494538240 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 125829120 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚322 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚4287 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚525758754490368 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 4617473198080 BDeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4289 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 845178880 ZLDeviceStatsMonitor.on_train_batch_startsQøNÊ/inactive_split_bytes.large_pool.peak‚ 831258624 PCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚58387380 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚16022510 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚ 134822777 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚41945088 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚61478912 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚41348020 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 $ _timestamp‚1.786663714699045e+09 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚64114063 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚223 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚525758754490368 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚262 @7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚5264 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚522668881330287 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 5368709120 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 F?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚60 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 5368709120 trainer/global_step‚19099 _runtime‚5063.232948269 KDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚60 JDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4876 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 4654602240 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 4552982528 KBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚4876 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 4607899675683 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 4607858327663 M@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚59049984 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17104896 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚44025195 UHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚70973440 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 122306526 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 5494538240 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚521141281292288 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 JBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚665 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚459168012475904 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚50329600 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 125829120 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 122905600 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚4287 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚322 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚64109776 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚525758813540352 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚521141281292288 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 4552982528 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚70713001 IBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚80 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 _step‚1572 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚44025197 C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚4289 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚70713001 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 4654602240 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17104896 JDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚16023309 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚518087225835520 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚ 134829617 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚4293 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚429 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚522695297696435 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚322 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 125829120 G@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚60 trainer/global_step‚19099 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚70716556 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 122905600 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚4293 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚44027400 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 4883800486400 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚322 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 125829120 B9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚5264 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 4883748062720 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚61478912 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 NADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚58387428 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 5368709120 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚525785366949888 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚521167603630080 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 4617704266752 I@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚4876 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚223 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚209 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654602240 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚70716554 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚20643840 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚454307485646848 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 4517085184 K>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59053056 PDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 SFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚73067520 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚262 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚521167620734976 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚64117356 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚64113063 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 5368709120 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 4617704266752 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚518087208796160 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 4608088900275 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 5494538240 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚16023388 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 845178880 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 D=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚60 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚521167603630080 B6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚322 IBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚60 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 4617746214912 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚522695356083863 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 5368709120 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 134833912 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 _runtime‚5063.394112927 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚44027402 EDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚60050709 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17104896 G@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚81 >5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5264 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚521167620734976 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚41348068 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚459191233709568 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 NGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚79 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 125829120 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚322 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚ 134833912 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚ 134829617 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 % _timestamp‚1.7866637148636389e+09 DDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521206 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚262µ<Ô&¼,ÎM E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚459191233709568 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 NGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚79 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 125829120 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚322 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚ 134833912 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚ 134829617 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 % _timestamp‚1.7866637148636389e+09 DDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521206 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚262 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚4295 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 5494538240 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚64117356 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 4654602240 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚525785307896832 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚525785366949888 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚60050790 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚459191306777088 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚64113063 EDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚16023309 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚518087225835520 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚ 134829617 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚4293 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚429 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚522695297696435 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚322 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 125829120 G@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚60 trainer/global_step‚19099 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚70716556 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 122905600 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚4293 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚44027400 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 4883800486400 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚322 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 125829120 B9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚5264 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 4883748062720 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚61478912 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 NADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚58387428 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 5368709120 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚525785366949888 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚521167603630080 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 4617704266752 I@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚4876 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚223 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚209 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654602240 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚70716554 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚20643840 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚454307485646848 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 4517085184 K>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59053056 PDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 SFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚73067520 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚262 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚521167620734976 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚64117356 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚64113063 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 5368709120 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 4617704266752 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚518087208796160 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 4608088900275 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 5494538240 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚16023388 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 845178880 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 D=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚60 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚521167603630080 B6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚322 IBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚60 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 4617746214912 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚522695356083863 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 5368709120 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 134833912 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 _runtime‚5063.394112927 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚44027402 EDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚60050709 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17104896 G@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚81 >5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5264 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚521167620734976 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚41348068 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0¥ À¡êo2½,- _wandbruntime‚5063  _runtime‚5063,ux¾,ò 1loss/FRt1/assignment_loss‚0.45194838956559397 0loss/FRt2/assignment_loss‚0.3537572114835332 0loss/FRt2/detection_loss‚0.46446961638495504 &loss/total_loss‚1.7569827178632182 epoch‚47 0loss/FRt1/detection_loss‚0.48680750042913606 trainer/global_step‚19099 % _timestamp‚1.7866637148651752e+09 _runtime‚5063.394979982 _step‚1574t€R¨}¿,÷ 0loss/FRt1/detection_loss‚0.48680750042913606 trainer/global_step‚19099 % _timestamp‚1.7866637148651752e+09 _runtime‚5063.394979982 _step‚1574 1loss/FRt1/assignment_loss‚0.45194838956559397 0loss/FRt2/assignment_loss‚0.3537572114835332 0loss/FRt2/detection_loss‚0.46446961638495504 &loss/total_loss‚1.7569827178632182 epoch‚47¦ šíA¥–À,‚(:‹ ©¦ùÓÀ‰ëà gpu.0.gpu‚63.0 gpu.0.memory‚12, gpu.0.memoryAllocated‚7.0313262939453125) gpu.0.memoryAllocatedBytes‚ 6039863296 gpu.0.temp‚36.0 gpu.0.powerWatts‚211.788' gpu.0.enforcedPowerLimitWatts‚500.0 gpu.0.powerPercent‚42.3576 gpu.0.smClock‚1410 gpu.0.memoryClock‚1593$ gpu.0.correctedMemoryErrors‚1955# gpu.0.uncorrectedMemoryErrors‚0¶làœÁ,‚(:‘ ©¦ùÓò²²ã proc.cpu.threads‚17 disk.sda3.in‚ 46908.2109375 disk.sda3.out‚ 93.64453125! disk.nvme0n1.in‚ 76245.0234375& disk.nvme0n1.out‚63321.65673828125 disk.dm-1.in‚ 46908.2109375 disk.dm-1.out‚ 93.64453125 disk.drbd2165.in‚0 disk.dm-0.in‚ 76245.0234375 disk.sda2.out‚0 network.sent‚ 2.4825812e+07* disk./.usagePercent‚32.694811032441926$ disk./.usageGB‚487.1763114929199( disk.nvme0c0n1.out‚63321.65673828125 disk.sda.in‚ 46908.2109375 disk.sda1.in‚0# disk.dm-0.out‚63321.65673828125# disk.nvme0c0n1.in‚ 76245.0234375 disk.sda1.out‚0# memory_percent‚68.5833215713501 cpu‚27.25869527304852 disk.sda2.in‚0 network.recv‚ 9.488424e+06 disk.sda.out‚ 93.64453125 disk.drbd2165.out‚0) proc.memory.availableMB‚ 5147.30859375# proc.memory.rssMB‚ 2346.43359375* proc.memory.percent‚14.321494102478027ëž;B2Â,- _wandbruntime‚5075  _runtime‚5075PÜÙ"ÊÃ,Ä ' lr-AdamW/pg1‚6.9663376811018865e-06 ' lr-AdamW/pg2‚6.9663376811018865e-06 trainer/global_step‚19149 % _timestamp‚1.7866637270266447e+09 _runtime‚ 5075.55665902 _step‚1575Fª(TÏÄ,É % _timestamp‚1.7866637270266447e+09 _runtime‚ 5075.55665902 _step‚1575 ' lr-AdamW/pg1‚6.9663376811018865e-06 ' lr-AdamW/pg2‚6.9663376811018865e-06 trainer/global_step‚19149§ _QX2Å,- _wandbruntime‚5075  _runtime‚5075é©Æ,¾O =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 4619461486291 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚322 C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚4289 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 4629101908992 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚64279165 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚16062985 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 125829120 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚522457398181888 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17104896 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚60198366 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 845178880 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚322 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 5494538240 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚262 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚527086517195776 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 4629101908992 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚20643840 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚70890653 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 4552982528 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚41945088 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 122905600 UHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚66779136 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 5368709120 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 4654602240 D;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚5264 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 4619420138271 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 5494538240 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 122905600 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚70890651 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚4287 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚455436827557888 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 831258624 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 4895828484608 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 5494538240 $ _timestamp‚1.786663727027574e+09 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚223 JDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚527086458145792 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚60198432 IBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚66 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 122306526 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚64274878 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 4629059963904 JBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚665 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 4654602240 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚209 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚460332589263360 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 4552982528 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4289 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚41348020 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚41945088 KBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚4876 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚460332656042496 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚429 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚16062921 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 5368709120 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚262 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 F>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 125829120 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚322 _step‚1576 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚ 135169818 JDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4876 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 4629059963904â$BÅÉ'Ç,ÃO F8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚ 135165529 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 4895782349312 PCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚58387380 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 F?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚60 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚527086458145792 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚60198432 IBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚66 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 122306526 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚64274878 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 4629059963904 JBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚665 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 4654602240 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚209 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚460332589263360 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 4552982528 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4289 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚41348020 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚41945088 KBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚4876 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚460332656042496 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚429 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚16062921 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 5368709120 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚262 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 F>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 125829120 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚322 _step‚1576 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚ 135169818 JDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4876 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 4629059963904 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 4619461486291 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚322 C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚4289 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 4629101908992 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚64279165 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚16062985 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 125829120 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚522457398181888 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17104896 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚60198366 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 845178880 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚322 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 5494538240 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚262 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚527086517195776 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 4629101908992 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚20643840 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚70890653 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 4552982528 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚41945088 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 122905600 UHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚66779136 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 5368709120 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 4654602240 D;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚5264 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 4619420138271 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 5494538240 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 122905600 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚70890651 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚4287 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚455436827557888 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 831258624 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 4895828484608 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 5494538240 $ _timestamp‚1.786663727027574e+09 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚223 JDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 135176664 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 4517085184 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 125829120 B9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚5264 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 831258624 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 EDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚60201369 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 K>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59053056 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚4293 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚50326528 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚322 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚262 >5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5264 NADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚59053056 H@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚665 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚44137650 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚209 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 % _timestamp‚1.7866637271982644e+09 EDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521206 D6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚322 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚41348068 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 122905600 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚70894206 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 845178880 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚527113011552256 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚16063786 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚20643840 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚524016207571271 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 5368709120 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 125829120 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚64282458 I@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚4876 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 5368709120 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚527113070605312 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 _step‚1577 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚519396498472960 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17104896 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 4629332980736 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚64278165 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 125829120 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚322 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚522483720519680 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚ 135172369 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚429 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚460355948082688 BDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 trainer/global_step‚19149 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚527113070605312 PDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654602240 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚522483737624576 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚41948160 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 4896073152000 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚524016149183843 G@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚60 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 4629291032576 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚455459874930688 NADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚58387428 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17104896 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 4552982528 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 122306526 _runtime‚5075.728792836 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 5368709120¿ÂD}øÊ,ÎM REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚64282458 I@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚4876 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 5368709120 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚527113070605312 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 _step‚1577 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚519396498472960 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17104896 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 4629332980736 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚64278165 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 125829120 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚322 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚522483720519680 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚ 135172369 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚429 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚460355948082688 BDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 trainer/global_step‚19149 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚527113070605312 PDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654602240 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚522483737624576 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚41948160 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 4896073152000 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚524016149183843 G@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚60 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 4629291032576 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚455459874930688 NADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚58387428 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17104896 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 4552982528 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 122306526 _runtime‚5075.728792836 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 5368709120 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚64278165 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 122905600 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚455459854286848 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚61478912 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚322 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 4619650710883 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 5494538240 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚ 135176664 G@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚69 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚70894204 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 4654602240 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚41948160 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚60201438 SFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚70970368 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚262 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚4293 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 135176664 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 4517085184 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 125829120 B9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚5264 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 831258624 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 EDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚60201369 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 K>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59053056 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚4293 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚50326528 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚322 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚262 >5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5264 NADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚59053056 H@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚665 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚44137650 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚209 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 % _timestamp‚1.7866637271982644e+09 EDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521206 D6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚322 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚41348068 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 122905600 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚70894206 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 845178880 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚527113011552256 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚16063786 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚20643840 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚524016207571271 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 5368709120 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 125829120© æÜ„l2Ë,- _wandbruntime‚5075  _runtime‚5075p{ uÌ,ï /loss/FRt1/detection_loss‚0.5062783415321062 0loss/FRt2/detection_loss‚0.43107186662322317 trainer/global_step‚19149 % _timestamp‚1.7866637271993258e+09 _runtime‚5075.729199042 _step‚1578 0loss/FRt1/assignment_loss‚0.4328522430940413 %loss/total_loss‚1.744307608370249 epoch‚47 0loss/FRt2/assignment_loss‚0.3741051571208783(‘J0zÍ,ô % _timestamp‚1.7866637271993258e+09 _runtime‚5075.729199042 _step‚1578 0loss/FRt1/assignment_loss‚0.4328522430940413 %loss/total_loss‚1.744307608370249 epoch‚47 0loss/FRt2/assignment_loss‚0.3741051571208783 /loss/FRt1/detection_loss‚0.5062783415321062 0loss/FRt2/detection_loss‚0.43107186662322317 trainer/global_step‚19149ª Ñn¿s¡Î,‚(:– ¸¦ùÓ‚¹ƒá gpu.0.gpu‚62.0 gpu.0.memory‚11, gpu.0.memoryAllocated‚7.0313262939453125) gpu.0.memoryAllocatedBytes‚ 6039863296 gpu.0.temp‚36.0 gpu.0.powerWatts‚143.688' gpu.0.enforcedPowerLimitWatts‚500.0) gpu.0.powerPercent‚28.737599999999997 gpu.0.smClock‚1410 gpu.0.memoryClock‚1593$ gpu.0.correctedMemoryErrors‚1955# gpu.0.uncorrectedMemoryErrors‚0qá0EšÏ,‚(: ¸¦ùÓ“åêâ network.recv‚ 9.521003e+06* disk./.usagePercent‚32.695253925476166 disk.drbd2165.out‚0 disk.sda3.in‚ 46908.2109375& disk.nvme0c0n1.out‚63334.458984375 disk.sda2.in‚0" disk.nvme0n1.in‚76248.20703125# memory_percent‚68.5884952545166 disk.drbd2165.in‚0 disk.sda3.out‚ 93.87109375$ disk.nvme0c0n1.in‚76248.20703125( proc.memory.availableMB‚ 5146.4609375# proc.memory.rssMB‚ 2346.43359375* proc.memory.percent‚14.321494102478027 cpu‚27.258187863127162% disk./.usageGB‚487.18291091918945 disk.dm-1.out‚ 93.87109375 disk.dm-0.in‚76248.20703125 disk.sda1.in‚0 disk.sda2.out‚0 proc.cpu.threads‚17 network.sent‚ 2.4900981e+07 disk.dm-1.in‚ 46908.2109375! disk.dm-0.out‚63334.458984375 disk.sda.in‚ 46908.2109375 disk.sda.out‚ 93.87109375 disk.sda1.out‚0$ disk.nvme0n1.out‚63334.458984375ó(2Ð,- _wandbruntime‚5088  _runtime‚5088QÉÑ,à & lr-AdamW/pg2‚6.172871813376807e-06 trainer/global_step‚19199 % _timestamp‚1.7866637397292233e+09 _runtime‚5088.259238305 _step‚1579 & lr-AdamW/pg1‚6.172871813376807e-06$â&ÎÒ,È & lr-AdamW/pg1‚6.172871813376807e-06 & lr-AdamW/pg2‚6.172871813376807e-06 trainer/global_step‚19199 % _timestamp‚1.7866637397292233e+09 _runtime‚5088.259238305 _step‚1579« t„±V2Ó,- _wandbruntime‚5088  _runtime‚5088å£tÅ'Ô,¿O L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚71068301 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚520679619395584 _step‚1580 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚322 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚523773532176384 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚520679602356224 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚456589196197888 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 PCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚58387380 D;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚5264 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 F>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 4552982528 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚16103931 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚528414161801216 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚523773515071488 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚523773532176384 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚61478912 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚322 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚20643840 trainer/global_step‚19199 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚528414220851200 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 _runtime‚5088.263886504 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 125829120 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚71068301 KBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚4876 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚209 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 845178880 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 BDeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4289 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚429 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 125829120 KDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚60 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 4654602240 IBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚72 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚525310584305103 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 122905600 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 831258624 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚322 G>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4876 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚16104001 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚456589175554048 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 4908057487360 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚525310642692483 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 4631023296899 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 5368709120 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚44245697 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚46135296 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚ 135508281 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚41945088 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 5494538240 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 5368709120 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚262 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚60349698 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚523773515071488 PIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚70 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 4628521206 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 122905600 JBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚665 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 4908103622656 C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚4289 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17104896 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 4552982528 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚64439980 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚41348020 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚71068303 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚461497233041408 IBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚60 % _timestamp‚1.7866637397301488e+09 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚41945088 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚ 135512570 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 5494538240 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 125829120 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚262 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚322 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 5368709120 M@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚59049984 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 4654602240 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 122306526 JDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4876 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚16104001 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚456589175554048 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 4908057487360 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚525310642692483 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 4631023296899 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 5368709120 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚44245697 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚46135296 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚ 135508281 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚41945088 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 5494538240 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 5368709120 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚262 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚60349698 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚523773515071488 PIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚70 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 4628521206 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 122905600 JBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚665 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 4908103622656 C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚4289 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17104896 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 4552982528 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚64439980 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚41348020 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚71068303 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚461497233041408 IBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚60 % _timestamp‚1.7866637397301488e+09 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚41945088 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚ 135512570 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 5494538240 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 125829120 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚262 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚322 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 5368709120 M@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚59049984 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 4654602240 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 122306526 JDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 4552982528 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚16103931 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚528414161801216 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚523773515071488 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚523773532176384 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚61478912 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚322 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚20643840 trainer/global_step‚19199 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚528414220851200 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 _runtime‚5088.263886504 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 125829120 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚71068301 KBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚4876 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚209 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 845178880 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 BDeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4289 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚429 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 125829120 KDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚60 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 4654602240 IBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚72 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚525310584305103 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 122905600 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 831258624 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚322¬ è‰2Ö,- _wandbruntime‚5088  _runtime‚5088ŒðFšª×,ÈM UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚41948160 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚44247902 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 125829120 >5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5264 H@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚665 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 125829120 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚322 IBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚60 BDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 % _timestamp‚1.7866637398943572e+09 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚4293 P8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚520705805189120 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 _runtime‚ 5088.42482497 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚528440774260736 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚322 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 4640919746560 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 845178880 NADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚58387428 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚64443267 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 5494538240 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚322 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17104896 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚528440774260736 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654602240 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚223 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚209 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 125829120 DDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521206 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 4631212521491 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚4295 EDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚60352653 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚525337059058679 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚64447560 I@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚4876 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 831258624 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚322 G@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚60 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚41948160 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 4640877798400 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚461520525445120 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚262 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚44247900 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚64443267 EDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59053056 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚523799854514176 _step‚1581 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 4640877798400 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚523799854514176 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚64447560 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚16104818 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚41348068 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 5494538240 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚71071856 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚262 NADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚59053056 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚16104753 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚20643840 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 4631253869559 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 5368709120 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 135519416 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚429 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚71071856 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚523799837409280 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚523799837409280ýy'1Ó&Ø,ÍM NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚64443267 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 5494538240 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚322 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17104896 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚528440774260736 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654602240 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚223 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚209 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 125829120 DDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521206 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 4631212521491 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚4295 EDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚60352653 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚525337059058679 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚64447560 I@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚4876 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 831258624 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚322 G@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚60 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚41948160 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 4640877798400 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚461520525445120 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚262 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚44247900 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚64443267 EDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59053056 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚523799854514176 _step‚1581 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 4640877798400 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚523799854514176 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚64447560 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚16104818 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚41348068 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 5494538240 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚71071856 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚262 NADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚59053056 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚16104753 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚20643840 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 4631253869559 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 5368709120 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 135519416 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚429 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚71071856 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚523799837409280 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚523799837409280 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚41948160 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚44247902 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 125829120 >5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5264 H@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚665 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 125829120 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚322 IBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚60 BDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 % _timestamp‚1.7866637398943572e+09 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚4293 P8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚520705805189120 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 _runtime‚ 5088.42482497 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚528440774260736 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚322 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 4640919746560 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 845178880 NADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚58387428­ ±—22Ù,- _wandbruntime‚5088  _runtime‚5088((.ÝvÚ,ð 1loss/FRt1/assignment_loss‚0.46515626422042444 /loss/FRt1/detection_loss‚0.5002120208519292 0loss/FRt2/detection_loss‚0.48069432057893874 epoch‚47 $ _timestamp‚1.786663739895405e+09 0loss/FRt2/assignment_loss‚0.3727199282041664 &loss/total_loss‚1.8187825338554586 trainer/global_step‚19199 _runtime‚5088.425274329 _step‚1582z£k{Û,õ 1loss/FRt1/assignment_loss‚0.46515626422042444 /loss/FRt1/detection_loss‚0.5002120208519292 0loss/FRt2/detection_loss‚0.48069432057893874 epoch‚47 $ _timestamp‚1.786663739895405e+09 0loss/FRt2/assignment_loss‚0.3727199282041664 &loss/total_loss‚1.8187825338554586 trainer/global_step‚19199 _runtime‚5088.425274329 _step‚1582® ¾Ø]Î2Ü,- _wandbruntime‚5093  _runtime‚5093gpܲÞÝ,Ø 3Purity/0FRt/event_proportion‚0.3301943838596344 +jet/accuracy_1_of_1‚0.30981114506721497 )jet/accuracy_1_of_2‚0.571577787399292 Purity/0FRt/event_purity‚1 2Purity/2FRt/event_proportion‚0.040062315762043 /particle/accuracy_2_of_2‚0.6836098432540894 +particle/sensitivity‚0.6320492625236511 +particle/specificity‚0.6429870128631592 6validation_average_jet_accuracy‚0.3146941363811493 /Purity/1FRt/event_purity‚0.4678609073162079 0Purity/2FRt/event_purity‚0.42591580748558044 +jet/accuracy_2_of_2‚0.21070505678653717 3Purity/1FRt/event_proportion‚0.6297432780265808 -Purity/2FRt/FRt_purity‚0.5629440546035767 trainer/global_step‚19199 )loss/val_total_loss‚2.546640213856042 particle/accuracy_1_of_1‚1 /particle/accuracy_1_of_2‚0.9248088002204895 _step‚1583 -Purity/*FRt/FRt_purity‚0.4786949157714844 /Purity/*FRt/event_purity‚0.4654359221458435 (particle/accuracy‚0.6391136646270752 'particle/f_score‚0.5542073249816895 % _timestamp‚1.7866637450731676e+09 _runtime‚5093.603195898 2Purity/*FRt/event_proportion‚0.669805645942688 -Purity/1FRt/FRt_purity‚0.4678609073162079 +validation_accuracy‚0.21070505678653717 epoch‚47Çâ}zãÞ,Ý -Purity/*FRt/FRt_purity‚0.4786949157714844 /Purity/*FRt/event_purity‚0.4654359221458435 (particle/accuracy‚0.6391136646270752 'particle/f_score‚0.5542073249816895 % _timestamp‚1.7866637450731676e+09 _runtime‚5093.603195898 2Purity/*FRt/event_proportion‚0.669805645942688 -Purity/1FRt/FRt_purity‚0.4678609073162079 +validation_accuracy‚0.21070505678653717 epoch‚47 3Purity/0FRt/event_proportion‚0.3301943838596344 +jet/accuracy_1_of_1‚0.30981114506721497 )jet/accuracy_1_of_2‚0.571577787399292 Purity/0FRt/event_purity‚1 2Purity/2FRt/event_proportion‚0.040062315762043 /particle/accuracy_2_of_2‚0.6836098432540894 +particle/sensitivity‚0.6320492625236511 +particle/specificity‚0.6429870128631592 6validation_average_jet_accuracy‚0.3146941363811493 /Purity/1FRt/event_purity‚0.4678609073162079 0Purity/2FRt/event_purity‚0.42591580748558044 +jet/accuracy_2_of_2‚0.21070505678653717 3Purity/1FRt/event_proportion‚0.6297432780265808 -Purity/2FRt/FRt_purity‚0.5629440546035767 trainer/global_step‚19199 )loss/val_total_loss‚2.546640213856042 particle/accuracy_1_of_1‚1 /particle/accuracy_1_of_2‚0.9248088002204895 _step‚1583¯ é¬IX–ß,‚(:‹ ǦùÓä»–á gpu.0.gpu‚64.0 gpu.0.memory‚12, gpu.0.memoryAllocated‚7.0313262939453125) gpu.0.memoryAllocatedBytes‚ 6039863296 gpu.0.temp‚34.0 gpu.0.powerWatts‚118.926' gpu.0.enforcedPowerLimitWatts‚500.0 gpu.0.powerPercent‚23.7852 gpu.0.smClock‚1410 gpu.0.memoryClock‚1593$ gpu.0.correctedMemoryErrors‚1955# gpu.0.uncorrectedMemoryErrors‚0E2Uà,‚(:’ ǦùÓ‡úÇã disk.sda.in‚ 46908.2109375 disk.sda.out‚ 94.06640625 disk.sda2.in‚0$ memory_percent‚68.63551139831543 disk.dm-1.in‚ 46908.2109375 disk.drbd2165.in‚0$ disk.nvme0c0n1.in‚76248.20703125 disk.sda1.in‚0 disk.sda1.out‚0" disk.nvme0n1.in‚76248.20703125& disk.nvme0n1.out‚63341.74365234375 cpu‚27.26282787836495) disk./.usagePercent‚32.69525238943096 disk.drbd2165.out‚0 disk.sda3.out‚ 94.06640625# proc.memory.rssMB‚ 2354.24609375% disk./.usageGB‚487.18288803100586 disk.dm-0.in‚76248.20703125 disk.sda3.in‚ 46908.2109375( disk.nvme0c0n1.out‚63341.74365234375 disk.sda2.out‚0( proc.memory.availableMB‚ 5138.7578125) proc.memory.percent‚14.36917781829834 proc.cpu.threads‚17 network.sent‚ 2.4968439e+07 network.recv‚ 9.54548e+06 disk.dm-1.out‚ 94.06640625# disk.dm-0.out‚63341.74365234375DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 D;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚5264 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 4646609731588 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚41945088 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 845178880 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚262 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚ 135933826 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 5494538240 UHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚68876288 PCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚58387380 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚322 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚262 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 PCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚59049984 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚64627269 trainer/global_step‚19249 % _timestamp‚1.7866637575694044e+09 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4289 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚522369791524864 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 4656343164416 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚60550063 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚44396559 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 125829120 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚64627269 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚525480586280960 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 4656343164416 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚60549992 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚48232448 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚61478912 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚522369808564224 G>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4876 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17104896 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚20643840 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚322 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚322 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 _step‚1585 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚530136870395392 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚41945088 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚527016401256452 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 4628521206 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 4517085184 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 4552982528 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚530136929445376 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 122905600 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 KDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚60 @7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚5264 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚525480586280960 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚525480569176064 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚16153433 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 5368709120rüˆoŒæ,ÄO XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚44396561 C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚4289 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚ 135938115 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚71306559 PIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚69 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 4654602240 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 4654602240 Jall_pool.peak‚ 125829120 F?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚60 IBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚71 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 5494538240 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚262 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 5368709120 F>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 D;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚5264 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 4646609731588 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚41945088 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 845178880 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚262 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚ 135933826 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 5494538240 UHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚68876288 PCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚58387380 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚322 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚262 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 PCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚59049984 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚64627269 trainer/global_step‚19249 % _timestamp‚1.7866637575694044e+09 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4289 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚522369791524864 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 4656343164416 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚60550063 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚44396559 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 125829120 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚64627269 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚525480586280960 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 4656343164416 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚60549992 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚48232448 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚61478912 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚522369808564224 G>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4876 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17104896 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚20643840 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚322 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚322 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 _step‚1585 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚530136870395392 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚41945088 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚527016401256452 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 4628521206 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 4517085184 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 4552982528 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚530136929445376 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 122905600 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 KDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚60 @7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚5264 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚525480586280960 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚525480569176064 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚16153433 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 5368709120 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚71306557 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚530136929445376 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚41348020 IBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚60 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0± Ô»rq2ç,- _wandbruntime‚5106  _runtime‚5106 MÛäÏ&è,ÉM T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚530163482854912 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654602240 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 4924739774976 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 trainer/global_step‚19249 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚209 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚522395977318400 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚64630556 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 4656532288000 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 5368709120 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 125829120 NADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚58387428 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚ 135944961 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 4552982528 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚71310110 H@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚665 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚4293 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 831258624 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚522395994357760 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚458063882518528 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 _runtime‚5106.270260623 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚16154319 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 4924695739904 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 5494538240 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚71310112 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 5494538240 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚41948160 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚429 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚525506891513856 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚223 SFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚64678912 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 B6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚322 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 4646840304200 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 122306526 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 IBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚60 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚ 135940666 NGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚60 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 % _timestamp‚1.7866637577398856e+09 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚4293 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 122905600 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚20643840 G@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚60 _step‚1586 B9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚5264 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 4656574236160 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚64634849 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚262 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚262 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 DDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521206 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚64630556 E6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚322 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 4552982528 EDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 135944961 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚462988622293504 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚530163482854912 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚4295 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚71310110 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚60553023 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚44398766 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 K>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59053056 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚527042817622600 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 5494538240 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 5368709120 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚41948160 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 4646881652268 D=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚60 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚ 135940666 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚525506908618752 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚44035072 >5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5264 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17104896 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 G@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚62 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚61478912 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚527042876010028 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 125829120 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚71310112 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 4656574236160 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚462988557614592 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 125829120 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚525506891513856“ÓÔ&é,ÎM HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚64630556 E6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚322 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 4552982528 EDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 135944961 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚462988622293504 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚530163482854912 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚4295 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚71310110 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚60553023 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚44398766 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 K>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59053056 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚527042817622600 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 5494538240 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 5368709120 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚41948160 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 4646881652268 D=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚60 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚ 135940666 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚525506908618752 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚44035072 >5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5264 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17104896 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 G@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚62 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚61478912 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚527042876010028 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 125829120 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚71310112 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 4656574236160 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚462988557614592 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 125829120 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚525506891513856 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚530163482854912 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654602240 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 4924739774976 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 trainer/global_step‚19249 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚209 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚522395977318400 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚64630556 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 4656532288000 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 5368709120 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 125829120 NADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚58387428 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚ 135944961 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 4552982528 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚71310110 H@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚665 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚4293 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 831258624 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚522395994357760 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚458063882518528 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 _runtime‚5106.270260623 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚16154319 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 4924695739904 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 5494538240 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚71310112 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 5494538240 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚41948160 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚429 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚525506891513856 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚223 SFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚64678912 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 B6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚322 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 4646840304200 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 122306526 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 IBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚60 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚ 135940666 NGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚60 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 % _timestamp‚1.7866637577398856e+09 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚4293 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 122905600 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚20643840 G@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚60 _step‚1586 B9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚5264 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 4656574236160 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚64634849 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚262 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚262 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 DDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521206² VO>2ê,- _wandbruntime‚5106  _runtime‚5106`ê‘×wë,ñ _step‚1587 0loss/FRt1/assignment_loss‚0.4388204009463949 0loss/FRt2/assignment_loss‚0.4118279038447915 &loss/total_loss‚1.7968904061996238 trainer/global_step‚19249 % _timestamp‚1.7866637577409887e+09 _runtime‚5106.270847923 0loss/FRt1/detection_loss‚0.47173621422595563 0loss/FRt2/detection_loss‚0.47450588718248177 epoch‚48Ö2g7|ì,ö 0loss/FRt1/assignment_loss‚0.4388204009463949 0loss/FRt2/assignment_loss‚0.4118279038447915 &loss/total_loss‚1.7968904061996238 trainer/global_step‚19249 % _timestamp‚1.7866637577409887e+09 _runtime‚5106.270847923 0loss/FRt1/detection_loss‚0.47173621422595563 0loss/FRt2/detection_loss‚0.47450588718248177 epoch‚48 _step‚1587³ ûÎ(•í,‚(:Š Ö¦ùÓÈ’®á gpu.0.gpu‚27.0 gpu.0.memory‚3, gpu.0.memoryAllocated‚7.0313262939453125) gpu.0.memoryAllocatedBytes‚ 6039863296 gpu.0.temp‚36.0 gpu.0.powerWatts‚216.117' gpu.0.enforcedPowerLimitWatts‚500.0 gpu.0.powerPercent‚43.2234 gpu.0.smClock‚1410 gpu.0.memoryClock‚1593$ gpu.0.correctedMemoryErrors‚1955# gpu.0.uncorrectedMemoryErrors‚0à±®î,‚(:… Ö¦ùÓðñä network.sent‚ 2.5037335e+07) disk./.usagePercent‚32.69524726928027 disk.drbd2165.out‚0 disk.sda3.out‚ 94.5390625$ disk.nvme0c0n1.in‚76248.20703125 disk.sda.in‚ 46908.2109375 disk.sda1.in‚0" disk.nvme0n1.out‚ 63345.5390625 disk.dm-0.in‚76248.20703125$ disk.nvme0c0n1.out‚ 63345.5390625" disk.nvme0n1.in‚76248.20703125 disk.dm-1.out‚ 94.5390625$ memory_percent‚68.63996982574463) proc.memory.availableMB‚ 5138.02734375 proc.cpu.threads‚17 disk.dm-0.out‚ 63345.5390625 disk.sda.out‚ 94.5390625 disk.sda1.out‚0 disk.sda2.in‚0 disk.dm-1.in‚ 46908.2109375) proc.memory.percent‚14.36917781829834 network.recv‚ 9.569443e+06% disk./.usageGB‚487.18281173706055 disk.drbd2165.in‚0 disk.sda3.in‚ 46908.2109375 disk.sda2.out‚0# proc.memory.rssMB‚ 2354.24609375 cpu‚27.26234776882789u¨2ï,- _wandbruntime‚5118  _runtime‚5118O$nÉð,à _runtime‚5118.920033051 _step‚1588 & lr-AdamW/pg1‚4.729311851748963e-06 & lr-AdamW/pg2‚4.729311851748963e-06 trainer/global_step‚19299 % _timestamp‚1.7866637703900027e+09#ÑnHÎñ,È & lr-AdamW/pg1‚4.729311851748963e-06 & lr-AdamW/pg2‚4.729311851748963e-06 trainer/global_step‚19299 % _timestamp‚1.7866637703900027e+09 _runtime‚5118.920033051 _step‚1588´ Š*çN2ò,- _wandbruntime‚5118  _runtime‚5118Å|Â6Fó,¿O IBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚77 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 4667929930240 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 122905600 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚41945088 JBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚665 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚459193203785728 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚523679098241024 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚531464633100800 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 122306526 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚41945088 PCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚58387380 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 845178880 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 125829120 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 125829120 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚526796703170560 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 4552982528 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚464129974310912 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 5368709120 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚322 IBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚60 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚526796686065664 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚71484209 PIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚75 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚20643840 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚41348020 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚ 136280867 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚16194924 % _timestamp‚1.7866637703909414e+09 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚4287 G>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4876 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚526796686065664 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚50329600 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚528337311131240 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚528337252743860 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 4658212890216 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 5494538240 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 125829120 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 KDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚60 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 4667887985152 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚61478912 KEDeviceStatsMonitor.on_train_batch_start/allocat2™ü†ion.large_pool.current‚2 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚64796658 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚60701735 UHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚70973440 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚262 trainer/global_step‚19299 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 4552982528 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚64792371 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚464129903337472 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 5368709120 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚209 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚523679081201664 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚322 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚262 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚71484209 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17104896 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚531464574050816 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚44506811 D;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚5264 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚71484207 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 4936720195584 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 4658171542196 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 4654602240 PCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚59049984 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚60701658 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚223 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 _step‚1589 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚322 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 4628521206 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 4517085184 C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚4289 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚64792371 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚16194849 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚459193183141888 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 5494538240 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚4287 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚44506809 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚ 136276578 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 4667929930240 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4289 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚262 M@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚59049984 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 122905600 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 4654602240 BDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 F?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚60 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚64796658 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚531464574050816 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚429 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚322 @7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚5264 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17104896 JDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚523679081201664 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚322 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚262 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚71484209 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17104896 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚531464574050816 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚44506811 D;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚5264 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚71484207 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 4936720195584 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 4658171542196 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 4654602240 PCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚59049984 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚60701658 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚223 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 _step‚1589 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚322 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 4628521206 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 4517085184 C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚4289 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚64792371 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚16194849 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚459193183141888 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 5494538240 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚4287 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚44506809 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚ 136276578 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 4667929930240 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4289 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚262 M@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚59049984 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 122905600 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 4654602240 BDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 F?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚60 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚64796658 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚531464574050816 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚429 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚322 @7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚5264 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17104896 JDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4876 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚526796686065664 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚50329600 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚528337311131240 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚528337252743860 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 4658212890216 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 5494538240 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 125829120 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 KDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚60 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 4667887985152 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0µ ¡àíF2õ,- _wandbruntime‚5119  _runtime‚5119~¥.Î&ö,ÈM _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚459216230514688 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 4658443462876 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 5368709120 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚71487762 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚64799951 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚531491127457280 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚64799951 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 136287713 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚16195716 D=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚60 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚4295 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚60704653 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 BDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521206 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚4293 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚526823025508352 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚528363727497436 NADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚58387428 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚523705284034560 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 4517085184 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 4658402114808 trainer/global_step‚19299 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚ 136283418 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚ 136287713 DDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚64795658 I@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚4876 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚71487762 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 4937016241152 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚41348068 _runtime‚5119.094383341 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚526823025508352 SFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚70970368 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚71487760 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 122905600 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 5494538240 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚322 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚262 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654602240 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 4654602240 NADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚59053056 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 4552982528 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚464153267399680 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚528363669110008 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚41948160 G@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚79 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 4936965914624 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚322 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 125829120 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚322 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚531491186510336 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚44509014 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 G@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚60 PDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59053056 H@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚665 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚16195639 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 5368709120 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 4668119053824 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚44509016 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 NGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚77 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚4293 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚223 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚20643840 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 4668119053824 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 831258624 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 >5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5264 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚64795658 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 122306526 IBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚60 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚531491186510336 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚61478912 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚523705266995200 EDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 136287713 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚16195716 D=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚60 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚4295 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚60704653 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 BDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521206 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚4293 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚526823025508352 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚528363727497436 NADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚58387428 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚523705284034560 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 4517085184 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 4658402114808 trainer/global_step‚19299 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚ 136283418 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚ 136287713 DDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚64795658 I@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚4876 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚71487762 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 4937016241152 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚41348068 _runtime‚5119.094383341 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚526823025508352 SFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚70970368 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚71487760 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 122905600 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 5494538240 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚322 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚262 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654602240 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 4654602240 NADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚59053056 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 4552982528 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚464153267399680 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚528363669110008 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚41948160 G@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚79 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 4936965914624 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚322 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 125829120 C;DeviceStatsMonitor.on_train_batch_end/seg®=‡? ment.all.allocated‚322 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚531491186510336 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚44509014 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 G@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚60 PDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59053056 H@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚665 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚16195639 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 5368709120 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 4668119053824 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚44509016 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 NGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚77 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚4293 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚223 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚20643840 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 4668119053824 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 831258624 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 >5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5264 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚64795658 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 122306526 IBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚60 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚531491186510336 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚61478912 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚523705266995200 EDeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4289 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 5368709120 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚262 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚64957473 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚44617061 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 845178880 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 4517085184 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚44617059 F>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚16235607 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚223 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚64961760 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 4679474750976 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚ 136623619 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚60852593 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 trainer/global_step‚19349 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚48232448 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 4654602240 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 4669774700824 IBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚60 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚465294619687424 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚524988370878464 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 5494538240 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 125829120 @7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚5264 JDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚41945088 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚4287 UHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚68876288 PCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚58387380 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚532792277706240 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17104896 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚529658104231268 PCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚59049984 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚71661857 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 4654602240 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 122905600 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 125829120 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚322 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 JDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4876 _runtime‚5131.385027668 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚71661857 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 4949047261696 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚4287 M@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚59049984 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 4552982528 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚61478912 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 5494538240 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚528112802955264 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚209 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 122306526 F?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚60©tuÊ'‚-ÄO `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚528112820060160 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 5494538240 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 5368709120 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚322 G>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4876 _runtime‚5131.385027668 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚71661857 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 4949047261696 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚4287 M@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚59049984 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 4552982528 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚61478912 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 5494538240 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚528112802955264 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚209 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 122306526 F?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚60 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 122905600 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚16235534 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 5368709120 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 4552982528 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4289 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 5368709120 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚262 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚64957473 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚44617061 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 845178880 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 4517085184 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚44617059 F>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚16235607 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚223 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚64961760 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 4679474750976 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚ 136623619 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚60852593 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 trainer/global_step‚19349 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚48232448 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 4654602240 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 4669774700824 IBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚60 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚465294619687424 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚524988370878464 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 5494538240 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 125829120 @7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚5264 JDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚41945088 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚4287 UHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚68876288 PCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚58387380 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚532792277706240 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17104896 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚529658104231268 PCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚59049984 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚71661857 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 4654602240 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 122905600 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 125829120 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚322 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 J6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚322 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 4517085184 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚ 136626170 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚429 BDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 136630465 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚44619264 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚223 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚460368619798528 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 4670005273484 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 125829120 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 125829120 trainer/global_step‚19349 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚262 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 122905600 PDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚60855616 NGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚74 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚20643840 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚61478912 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚ 136626170 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚528139125293056 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚528139142397952 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚71665410 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚16236426 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚460368599154688 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 G@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚60 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚ 136630465 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 4552982528 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 _runtime‚5131.545540168 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚528139142397952 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚16236352 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 DDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 5494538240 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚4293 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 5494538240 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 5368709120 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚71665410 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚44619266 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚465317839348224 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 4949292617216 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 125829120 K>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59053056 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚41948160 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 4679747767808 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521206 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚64965053 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚465317912415744 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚322 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚41348068 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17104896 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚532818831112704 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654602240 E5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5264$HkUÔ&…-ÎM \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚465317912415744 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚322 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚41348068 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17104896 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚532818831112704 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654602240 E5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5264 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚71665412 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚52423680 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 4654602240 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚322 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 4517085184 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚ 136626170 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚429 BDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 136630465 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚44619264 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚223 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚460368619798528 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 4670005273484 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 125829120 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 125829120 trainer/global_step‚19349 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚262 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 122905600 PDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚60855616 NGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚74 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚20643840 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚61478912 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚ 136626170 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚528139125293056 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚528139142397952 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚71665410 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚16236426 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚460368599154688 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 G@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚60 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚ 136630465 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 4552982528 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 _runtime‚5131.545540168 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚528139142397952 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚16236352 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 DDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 5494538240 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚4293 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 5494538240 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 5368709120 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚71665410 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚44619266 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚465317839348224 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 4949292617216 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 125829120 K>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59053056 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚41948160 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 4679747767808 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521206 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚64965053 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2º ‡_ê 2†-- _wandbruntime‚5131  _runtime‚5131Šª¸Uw‡-ñ _step‚1595 0loss/FRt1/assignment_loss‚0.5660949305648338 /loss/FRt1/detection_loss‚0.5282309525143585 0loss/FRt2/detection_loss‚0.46508553088173255 &loss/total_loss‚1.9310022004339369 epoch‚48 1loss/FRt2/assignment_loss‚0.37159078647301197 trainer/global_step‚19349 % _timestamp‚1.7866637830162642e+09 _runtime‚5131.546074508s¾Œi|ˆ-ö &loss/total_loss‚1.9310022004339369 epoch‚48 1loss/FRt2/assignment_loss‚0.37159078647301197 trainer/global_step‚19349 % _timestamp‚1.7866637830162642e+09 _runtime‚5131.546074508 _step‚1595 0loss/FRt1/assignment_loss‚0.5660949305648338 /loss/FRt1/detection_loss‚0.5282309525143585 0loss/FRt2/detection_loss‚0.46508553088173255» mÇ!Ï2‰-- _wandbruntime‚5144  _runtime‚5144'K©ôÉŠ-à & lr-AdamW/pg2‚3.477218190282655e-06 trainer/global_step‚19399 % _timestamp‚1.7866637955697088e+09 _runtime‚5144.099828811 _step‚1596 & lr-AdamW/pg1‚3.477218190282655e-06ô^VÇ΋-È & lr-AdamW/pg1‚3.477218190282655e-06 & lr-AdamW/pg2‚3.477218190282655e-06 trainer/global_step‚19399 % _timestamp‚1.7866637955697088e+09 _runtime‚5144.099828811 _step‚1596¼ ñ^Š2Œ-- _wandbruntime‚5144  _runtime‚5144ÜG,Å'-¿O ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 5368709120 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚65126862 IBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚81 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚429 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚41348020 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚65122575 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚534119981361664 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 4654602240 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4289 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 KDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚60 F>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚41945088 JDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 125829120 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚71839509 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 4961332042240 C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚4289 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚529428936949760 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚61003642 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚209 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 5494538240 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 PCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚59049984 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚71839507 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 4517085184 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 125829120 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚4287 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 4691061516800 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 122905600 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚322 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚534119981361664 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 4552982528 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 831258624 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚530978955718676 M@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚59049984 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚529428936949760 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚44727311 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚466459273107968 PCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚58387380 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚526297660555264 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 _runtime‚5144.104906936 KBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚4876 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 4681295163412 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 4654602240 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚4287 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚262 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17104896 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 122905600 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚65122575 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚50329600 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚322 _step‚1597 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 5494538240 @7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚5264 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚44727309 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 5368709120 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17104896 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚71839509 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 4552982528 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 4961281712640 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 4681336511432 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 % _timestamp‚1.7866637955707896e+09 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚71839507 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚461497920421888 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 122306526 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚262 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 JBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚665 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚65126862 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 4628521206 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 125829120 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚41945088 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚16276412 UHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚70973440 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 5368709120 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 JDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4876 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 4691061516800 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚530979014106056 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚262†øªpŽ-ÄO `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚529428936949760 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚44727311 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚466459273107968 PCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚58387380 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚526297660555264 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 _runtime‚5144.104906936 KBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚4876 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 4681295163412 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 4654602240 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚4287 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚262 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17104896 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 122905600 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚65122575 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚50329600 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚322 _step‚1597 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 5494538240 @7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚5264 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚44727309 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 5368709120 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17104896 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚71839509 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 4552982528 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 4961281712640 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 4681336511432 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 % _timestamp‚1.7866637955707896e+09 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚71839507 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚461497920421888 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 122306526 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚262 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 JBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚665 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚65126862 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 4628521206 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 125829120 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚41945088 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚16276412 UHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚70973440 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 5368709120 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 JDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4876 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 4691061516800 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚530979014106056 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚262 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 5368709120 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚65126862 IBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚81 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚429 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚41348020 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚65122575 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚534119981361664 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 4654602240 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4289 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 KDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚60 F>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚41945088 JDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 125829120 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚71839509 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 4961332042240 C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚4289 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚529428936949760 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚61003642 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚209 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 5494538240 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 PCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚59049984 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚71839507 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 4517085184 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 125829120 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚4287 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 4691061516800 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 122905600 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚322 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚534119981361664 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 4552982528 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 831258624 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚530978955718676 M@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚59049984½ …)Z2-- _wandbruntime‚5144  _runtime‚5144Ù4œ®Î&-ÈM OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚61006709 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚322 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚529455259287552 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚531005430472252 IBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚60 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚71843062 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 4691334533632 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚ 136968922 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚65130155 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚526323846348800 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 4517085184 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 4552982528 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚71843060 $ _timestamp‚1.786663795747449e+09 H@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚665 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚429 DDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚61006639 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚44729514 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚534146534768128 SFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚66776064 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 I@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚4876 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚461520988438528 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 831258624 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚262 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚262 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚4295 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚4293 B5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5264 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚71843062 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚65125862 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 125829120 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚44729516 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚526323863388160 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 4681567084092 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 4681525736024 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 5368709120 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚534146593821184 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚531005372084824 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 5494538240 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 122905600 G@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚70 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚4293 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 P8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 NGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚68 _step‚1598 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 4552982528 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚61478912 NADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚58387428 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 5368709120 EDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 136973217 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚466482498372096 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚20643840 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654602240 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚322 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚65125862 K>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59053056 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚529455242182656 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17104896 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 4691292585472 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚41348068 _runtime‚5144.277908255 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚ 136968922 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚65130155 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 5494538240 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 5368709120 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521206 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚534146593821184 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚71843060 +Œ5Ó&‘-ÍM VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 5368709120 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚534146593821184 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚531005372084824 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 5494538240 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 122905600 G@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚70 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚4293 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 P8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 NGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚68 _step‚1598 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 4552982528 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚61478912 NADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚58387428 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 5368709120 EDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 136973217 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚466482498372096 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚20643840 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654602240 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚322 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚65125862 K>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59053056 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚529455242182656 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17104896 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 4691292585472 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚41348068 _runtime‚5144.277908255 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚ 136968922 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚65130155 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 5494538240 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 5368709120 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521206 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚534146593821184 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚71843060 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚61006709 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚322 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚529455259287552 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚531005430472252 IBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚60 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚71843062 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 4691334533632 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚ 136968922 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚65130155 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚526323846348800 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 4517085184 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 4552982528 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚71843060 $ _timestamp‚1.786663795747449e+09 H@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚665 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚429 DDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚61006639 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚44729514 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚534146534768128 SFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚66776064 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 I@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚4876 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚461520988438528 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 831258624 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚262 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚262 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚4295 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚4293 B5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5264 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚71843062 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚65125862 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 125829120 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚44729516 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚526323863388160 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 4681567084092 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 4681525736024¾ zà¼2’-- _wandbruntime‚5144  _runtime‚5144î–¨¸u“-ï /loss/FRt2/detection_loss‚0.4325573551152533 &loss/total_loss‚1.7324916598598854 epoch‚48 trainer/global_step‚19399 % _timestamp‚1.7866637957485328e+09 _step‚1599 0loss/FRt1/assignment_loss‚0.4689458064466462 /loss/FRt1/detection_loss‚0.4914886304409766 _runtime‚ 5144.27837648 1loss/FRt2/assignment_loss‚0.33949986785700936[ïâÏz”-ô 0loss/FRt1/assignment_loss‚0.4689458064466462 /loss/FRt1/detection_loss‚0.4914886304409766 _runtime‚ 5144.27837648 1loss/FRt2/assignment_loss‚0.33949986785700936 /loss/FRt2/detection_loss‚0.4325573551152533 &loss/total_loss‚1.7324916598598854 epoch‚48 trainer/global_step‚19399 % _timestamp‚1.7866637957485328e+09 _step‚1599¿ 0W¦¡•-‚(:– ô¦ùÓðÅ‹á gpu.0.gpu‚57.0 gpu.0.memory‚10, gpu.0.memoryAllocated‚7.0313262939453125) gpu.0.memoryAllocatedBytes‚ 6039863296 gpu.0.temp‚35.0 gpu.0.powerWatts‚125.204' gpu.0.enforcedPowerLimitWatts‚500.0) gpu.0.powerPercent‚25.040799999999997 gpu.0.smClock‚1410 gpu.0.memoryClock‚1593$ gpu.0.correctedMemoryErrors‚1955# gpu.0.uncorrectedMemoryErrors‚0ÆeŠŽ–-‚(:ƒ ô¦ùÓæÏã disk.sda.in‚46925.21484375 disk.sda1.in‚0 disk.sda2.in‚0" disk.nvme0n1.in‚76260.56640625! disk.nvme0n1.out‚ 63358.421875) proc.memory.percent‚14.36917781829834 network.sent‚ 2.5185017e+07) disk./.usagePercent‚32.69949085016785 disk.sda.out‚ 95.109375 disk.sda2.out‚0 disk.dm-1.in‚46925.21484375 disk.drbd2165.out‚0 disk.dm-0.in‚76260.56640625$ memory_percent‚68.64006519317627 disk.sda1.out‚0 disk.dm-1.out‚ 95.109375 disk.drbd2165.in‚0 disk.dm-0.out‚ 63358.421875# proc.memory.rssMB‚ 2354.24609375 cpu‚27.261654546036134 proc.cpu.threads‚17% disk./.usageGB‚487.24604415893555$ disk.nvme0c0n1.in‚76260.56640625# disk.nvme0c0n1.out‚ 63358.421875 disk.sda3.in‚46925.21484375 disk.sda3.out‚ 95.109375) proc.memory.availableMB‚ 5138.01171875 network.recv‚ 9.631505e+06I6*Ó2—-- _wandbruntime‚5157  _runtime‚5157'œ27˘-Å _runtime‚5157.151064592 _step‚1600 ' lr-AdamW/pg1‚2.9230736688444226e-06 ' lr-AdamW/pg2‚2.9230736688444226e-06 trainer/global_step‚19449 % _timestamp‚1.7866638086209483e+09|…‡fЙ-Ê % _timestamp‚1.7866638086209483e+09 _runtime‚5157.151064592 _step‚1600 ' lr-AdamW/pg1‚2.9230736688444226e-06 ' lr-AdamW/pg2‚2.9230736688444226e-06 trainer/global_step‚19449À ËÂf­2š-- _wandbruntime‚5157  _runtime‚5157A‹MV"›-¿O QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 5494538240 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 5368709120 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚322 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 5494538240 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚61478912 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 4517085184 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚535447685017088 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 122905600 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 122905600 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚16317100 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚20643840 JDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚532299807206084 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 4628521206 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚72017157 M@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚59049984 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚4287 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚530745053839360 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚209 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚41348020 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 4692856974020 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 5368709120 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 125829120 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚530745036734464 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚530745036734464 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 4654602240 JBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚665 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 831258624 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚467623895918080 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚462650289061888 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 BDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4876 D;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚5264 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚532299865593464 C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚4289 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚535447744067072 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 4702648282624 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚72017157 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚462650309705728 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4289 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 PCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚59049984 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚4287 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚530745053839360 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚41945088 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚527606950232064 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 125829120 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 125829120 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚ 137304834 PIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚79 IBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚60 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚322 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 4692898322040 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚322 @7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚5264 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚72017159 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚65287677 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚72017159 F>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚61154659 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚65291964 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚535447685017088 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 4702690227712 UHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚68876288 =6DeviceStatsMonitor.on_train_batch_start/max_split_Ö*Li£ size‚-1 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 KDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚60 JDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚535447685017088 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 122905600 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 122905600 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚16317100 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚20643840 JDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚532299807206084 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 4628521206 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚72017157 M@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚59049984 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚4287 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚530745053839360 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚209 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚41348020 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 4692856974020 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 5368709120 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 125829120 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚530745036734464 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚530745036734464 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 4654602240 JBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚665 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 831258624 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚467623895918080 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚462650289061888 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 BDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4876 D;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚5264 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚532299865593464 C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚4289 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚535447744067072 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 4702648282624 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚72017157 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚462650309705728 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4289 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 PCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚59049984 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚4287 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚530745053839360 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚41945088 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚527606950232064 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 125829120 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 125829120 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚ 137304834 PIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚79 IBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚60 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚322 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 4692898322040 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚322 @7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚5264 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚72017159 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚65287677 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚72017159 F>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚61154659 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚65291964 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚535447685017088 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 4702690227712 UHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚68876288 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 KDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚60 J6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚322 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚65290964 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 4552982528 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚16317962 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚462673336434688 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚532326281959660 D=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚60 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 845178880 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚527633136025600 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 4702879351296 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚72020712 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚61157664 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚209 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚535474297476608 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 4654602240 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 4702879351296 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17104896 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚65295257 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 4517085184 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚4293 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521206 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 PDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 137315969 I@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚4876 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 4973901492736 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 5368709120 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚322 E6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚322 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 5368709120 trainer/global_step‚19449 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 125829120 >5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5264 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 4702921299456 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚41948160 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 4693087546632 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚262 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 125829120 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚530771359072256 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 122905600 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚467647189698048 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 831258624 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚527633153064960 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 4693128894700 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 5368709120 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚44839764 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚535474297476608 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 4552982528 BDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 122306526 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 5494538240 G@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚60 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 4702921299456 E8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚72020710 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚322 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚262 IBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚60 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚530771376177152 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 125829120 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚ 137311674 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚535474238423552 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚48229376 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚532326223572232 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 5494538240 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654602240 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚ 137311674 H@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚665 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 4973853263360 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 _runtime‚5157.323466096 _step‚1602 K>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59053056 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚72020710 NADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚58387428IU}p&Ÿ-ÍM J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 4654602240 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 4702879351296 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17104896 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚65295257 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 4517085184 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚4293 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521206 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 PDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 137315969 I@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚4876 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 4973901492736 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 5368709120 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚322 E6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚322 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 5368709120 trainer/global_step‚19449 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 125829120 >5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5264 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 4702921299456 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚41948160 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 4693087546632 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚262 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 125829120 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚530771359072256 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 122905600 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚467647189698048 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 831258624 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚527633153064960 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 4693128894700 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 5368709120 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚44839764 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚535474297476608 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 4552982528 BDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 122306526 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 5494538240 G@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚60 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 4702921299456 E8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚72020710 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚322 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚262 IBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚60 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚530771376177152 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 125829120 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚ 137311674 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚535474238423552 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚48229376 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚532326223572232 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 5494538240 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654602240 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚ 137311674 H@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚665 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 4973853263360 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 _runtime‚5157.323466096 _step‚1602 K>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59053056 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚72020710 NADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚58387428 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚61157728 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚429 NGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚62 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚467647258571264 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚322 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚65290964 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 4552982528 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚16317962 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚462673336434688 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚532326281959660 D=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚60 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 845178880 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚527633136025600 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 4702879351296 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚72020712 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚61157664 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚209 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.f=ý›creed‚0 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚535474297476608 t•ñ»2 -- _wandbruntime‚5157  _runtime‚5157ÓŸÔ§u¡-ï &loss/total_loss‚1.7357793069966405 trainer/global_step‚19449 1loss/FRt2/assignment_loss‚0.35999385956902774 0loss/FRt1/detection_loss‚0.49255919889972727 epoch‚48 $ _timestamp‚1.786663808794191e+09 _runtime‚5157.324003451 _step‚1603 /loss/FRt1/assignment_loss‚0.439728519320798 /loss/FRt2/detection_loss‚0.4434977292070877“tùz¢-ô epoch‚48 $ _timestamp‚1.786663808794191e+09 _runtime‚5157.324003451 _step‚1603 /loss/FRt1/assignment_loss‚0.439728519320798 /loss/FRt2/detection_loss‚0.4434977292070877 &loss/total_loss‚1.7357793069966405 trainer/global_step‚19449 1loss/FRt2/assignment_loss‚0.35999385956902774 0loss/FRt1/detection_loss‚0.49255919889972727à j¬}O £-‚(:• ƒ§ùÓÛߟá gpu.0.gpu‚41.0 gpu.0.memory‚6, gpu.0.memoryAllocated‚7.0313262939453125) gpu.0.memoryAllocatedBytes‚ 6039863296 gpu.0.temp‚36.0 gpu.0.powerWatts‚179.582' gpu.0.enforcedPowerLimitWatts‚500.0) gpu.0.powerPercent‚35.916399999999996 gpu.0.smClock‚1410 gpu.0.memoryClock‚1593$ gpu.0.correctedMemoryErrors‚1955# gpu.0.uncorrectedMemoryErrors‚0ˆýšÖŒ¤-‚(: ƒ§ùÓ‰ÕŸã disk.sda1.in‚0 disk.sda1.out‚0 disk.sda2.in‚0 disk.nvme0n1.in‚ 76261.84375 disk.drbd2165.in‚0 disk.sda.out‚ 95.3828125% disk./.usageGB‚487.23796463012695 disk.dm-1.out‚ 95.3828125 disk.dm-0.in‚ 76261.84375 disk.sda3.in‚ 47844.953125 disk.sda.in‚ 47844.953125# proc.memory.rssMB‚ 2354.24609375) proc.memory.percent‚14.36917781829834 cpu‚27.2611919600873& disk.nvme0n1.out‚63406.85791015625 disk.dm-1.in‚ 47844.953125( disk.nvme0c0n1.out‚63406.85791015625$ memory_percent‚68.63899230957031% proc.memory.availableMB‚ 5138.1875 network.recv‚ 9.654169e+06 disk.sda2.out‚0 disk.drbd2165.out‚0# disk.dm-0.out‚63406.85791015625 disk.sda3.out‚ 95.3828125! disk.nvme0c0n1.in‚ 76261.84375 proc.cpu.threads‚17 network.sent‚ 2.5271441e+07* disk./.usagePercent‚32.698948626210324¨»)Ê2¥-- _wandbruntime‚5169  _runtime‚5169sµz˦-Å ' lr-AdamW/pg1‚2.4169125023573157e-06 ' lr-AdamW/pg2‚2.4169125023573157e-06 trainer/global_step‚19499 % _timestamp‚1.7866638210684605e+09 _runtime‚5169.598456514 _step‚1604¤\›ÝЧ-Ê ' lr-AdamW/pg2‚2.4169125023573157e-06 trainer/global_step‚19499 % _timestamp‚1.7866638210684605e+09 _runtime‚5169.598456514 _step‚1604 ' lr-AdamW/pg1‚2.4169125023573157e-06Ä *Oe´2¨-- _wandbruntime‚5169  _runtime‚5169?›ƒUÄ'©-¾O WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚41945088 JDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚322 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚532061170728960 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 4714235048448 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚536775447722496 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 5368709120 PCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚59049984 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 845178880 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚533620717080872 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 122306526 F?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚60 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚44947811 PCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚58387380 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 4654602240 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚72194809 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚468788637262336 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚41348020 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 BDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4876 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚16358277 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚533620658693492 F>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 122905600 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚536775388672512 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17104896 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚532061153624064 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 4552982528 D;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚5264 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚463802657701888 @7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚5264 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 4704460132648 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 5494538240 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚468788570483200 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 125829120 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 KDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚60 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚72194807 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚65457066 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚223 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚16358344 PIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚67 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 4985912781312 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚262 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 122905600 KBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚4876 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚532061153624064 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚4287 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4289 UHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚66779136 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 5368709120 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚ 137647586 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 5494538240 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 % _timestamp‚1.7866638210693557e+09 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚65452779 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚429 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚262 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚61306155 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚44947809 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚46135296 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 5494538240 JDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚536775388672512 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚209 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚463802678345728 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚322 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚4287 M@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚59049984 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚72194807 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 4517085184 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚322 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 4714235048448 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 125829120 _runtime‚ 5169.60288407 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚72194809 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚532061170728960 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚65457066 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 _step‚1605 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 4714276993536 JBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚665 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 4985958916608 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚528916239908864 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17104896 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚61478912 trainer/global_step‚19499 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚ 137651875 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚528916256948224 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0£mÔÉ'ª-ÃO O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 4628521206 IBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚69 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 831258624 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚322 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚532061170728960 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 4714235048448 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚536775447722496 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 5368709120 PCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚59049984 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 845178880 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚533620717080872 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 122306526 F?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚60 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚44947811 PCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚58387380 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 4654602240 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚72194809 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚468788637262336 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚41348020 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 BDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4876 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚16358277 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚533620658693492 F>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 122905600 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚536775388672512 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17104896 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚532061153624064 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 4552982528 D;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚5264 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚463802657701888 @7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚5264 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 4704460132648 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 5494538240 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚468788570483200 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 125829120 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 KDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚60 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚72194807 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚65457066 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚223 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚16358344 PIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚67 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 4985912781312 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚262 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 122905600 KBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚4876 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚532061153624064 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚4287 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4289 UHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚66779136 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 5368709120 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚ 137647586 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 5494538240 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 % _timestamp‚1.7866638210693557e+09 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚65452779 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚429 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚262 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚61306155 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚44947809 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚46135296 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 5494538240 JDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚536775388672512 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚209 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚463802678345728 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚322 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚4287 M@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚59049984 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚72194807 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 4517085184 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚322 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 4714235048448 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 125829120 _runtime‚ 5169.60288407 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚72194809 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚532061170728960 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚65457066 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 _step‚1605 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 4714276993536 JBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚665 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 4985958916608 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚528916239908864 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17104896 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚61478912 trainer/global_step‚19499 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚ 137651875 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚528916256948224 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚41945088 JDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 137658721 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚262 $ _timestamp‚1.786663821231755e+09 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚4293 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17104896 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚72198362 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚61309158 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17104896 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 4714508065280 G@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚73 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 125829120 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 4552982528 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚41948160 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 5368709120 NADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚59053056 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚463825725718528 >5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5264 EDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521206 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚223 DDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚322 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚262 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 4552982528 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚72198360 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚209 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 831258624 NADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚58387428 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚262 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚65460359 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚536802001132032 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚536802001132032 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚528942442741760 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 5368709120 IBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚60 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚532087475961856 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚65456066 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚ 137658721 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚468811927369216 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 125829120 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 NGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚71 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚20643840 B8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚528942425702400 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 4654602240 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚532087475961856 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚44035072 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 125829120 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚65456066 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654602240 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚65460359 I@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚4876 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚468811862690304 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 H@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚665 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚4293 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚533647133447068 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 5494538240 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 5368709120 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚ 137654426 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚61309085 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚16359071 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 5494538240 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 4714466117120 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚41348068 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 4704649357240 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 5494538240 _step‚1606 B9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚5264 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 D=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚60 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚72198360 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚ 137654426 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 4986201650688 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚322 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 4517085184 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚72198362 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 4714466117120 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚16359142 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚322 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚4295 GADeviceStatsMonitor(`.on_train_batch_end/oversize_segments.allocated‚0 K>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59053056 SFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚64678912 _runtime‚ 5169.76204244 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚532087493066752 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2lžRªÒ&­-ÌM [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚532087493066752 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚463825705074688 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 4714508065280 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚532087493066752 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 trainer/global_step‚19499 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚41948160 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 137658721 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚262 $ _timestamp‚1.786663821231755e+09 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚4293 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17104896 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚72198362 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚61309158 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17104896 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 4714508065280 G@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚73 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 125829120 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 4552982528 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚41948160 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 5368709120 NADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚59053056 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚463825725718528 >5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5264 EDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521206 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚223 DDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚322 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚262 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 4552982528 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚72198360 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚209 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 831258624 NADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚58387428 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚262 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚65460359 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚536802001132032 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚536802001132032 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚528942442741760 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 5368709120 IBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚60 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚532087475961856 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚65456066 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚ 137658721 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚468811927369216 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 125829120 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 NGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚71 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚20643840 B8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚528942425702400 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 4654602240 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚532087475961856 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚44035072 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 125829120 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚65456066 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654602240 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚65460359 I@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚4876 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚468811862690304 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 H@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚665 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚4293 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚533647133447068 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 5494538240 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 5368709120 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚ 137654426 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚61309085 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚16359071 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 5494538240 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 4714466117120 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚41348068 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 4704649357240 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 5494538240 _step‚1606 B9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚5264 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 D=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚60 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚72198360 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚ 137654426 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 4986201650688 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚322 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 4517085184 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚72198362 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 4714466117120 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚16359142 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚322 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚4295 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 K>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59053056 SFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚64678912 _runtime‚ 5169.76204244Æ Â¡W!2®-- _wandbruntime‚5169  _runtime‚5169)€‡w¯-ñ /loss/FRt1/detection_loss‚0.5040852722016392 &loss/total_loss‚1.8281757722771004 1loss/FRt1/assignment_loss‚0.44074212979997096 1loss/FRt2/assignment_loss‚0.39403322024614723 0loss/FRt2/detection_loss‚0.48931515002934284 epoch‚48 trainer/global_step‚19499 $ _timestamp‚1.786663821232681e+09 _runtime‚5169.762467403 _step‚1607#,+Ø|°-ö 1loss/FRt1/assignment_loss‚0.44074212979997096 1loss/FRt2/assignment_loss‚0.39403322024614723 0loss/FRt2/detection_loss‚0.48931515002934284 epoch‚48 trainer/global_step‚19499 $ _timestamp‚1.786663821232681e+09 _runtime‚5169.762467403 _step‚1607 /loss/FRt1/detection_loss‚0.5040852722016392 &loss/total_loss‚1.8281757722771004Ç £j¶Ÿ–±-‚(:‹ ’§ùÓ‚á¢á gpu.0.gpu‚59.0 gpu.0.memory‚11, gpu.0.memoryAllocated‚7.0313262939453125) gpu.0.memoryAllocatedBytes‚ 6039863296 gpu.0.temp‚36.0 gpu.0.powerWatts‚128.968' gpu.0.enforcedPowerLimitWatts‚500.0 gpu.0.powerPercent‚25.7936 gpu.0.smClock‚1410 gpu.0.memoryClock‚1593$ gpu.0.correctedMemoryErrors‚1955# gpu.0.uncorrectedMemoryErrors‚0õšë‚²-‚(:÷ ’§ùÓ‡‡¨ã network.recv‚ 9.67785e+06 disk.dm-0.in‚76263.25) proc.memory.percent‚14.36917781829834 cpu‚27.26082328489729 network.sent‚ 2.5338452e+07 disk.sda3.out‚ 95.51953125' disk.nvme0c0n1.out‚63450.8466796875 disk.sda1.out‚0 disk.dm-1.out‚ 95.51953125 disk.drbd2165.in‚0# memory_percent‚68.6391830444336# proc.memory.rssMB‚ 2354.24609375* disk./.usagePercent‚32.703681181488285$ disk./.usageGB‚487.3084831237793 disk.sda.in‚ 47844.953125 disk.sda.out‚ 95.51953125 disk.sda1.in‚0 disk.sda2.in‚0% disk.nvme0n1.out‚63450.8466796875 disk.drbd2165.out‚0 disk.sda3.in‚ 47844.953125 disk.nvme0c0n1.in‚76263.25 disk.sda2.out‚0 disk.nvme0n1.in‚76263.25 disk.dm-1.in‚ 47844.953125" disk.dm-0.out‚63450.8466796875& proc.memory.availableMB‚ 5138.15625 proc.cpu.threads‚17óêH/2³-- _wandbruntime‚5182  _runtime‚5182C´½È´- % _timestamp‚1.7866638337489228e+09 _runtime‚ 5182.27895724 _step‚1608 & lr-AdamW/pg1‚1.958767200607281e-06 & lr-AdamW/pg2‚1.958767200607281e-06 trainer/global_step‚19549Žö»šÍµ-Ç _runtime‚ 5182.27895724 _step‚1608 & lr-AdamW/pg1‚1.958767200607281e-06 & lr-AdamW/pg2‚1.958767200607281e-06 trainer/global_step‚19549 % _timestamp‚1.7866638337489228e+09È osãð2¶-- _wandbruntime‚5182  _runtime‚5182â· Å'·-¿O KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 BDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚322 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 4552982528 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 4725821814272 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚46135296 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 @7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚5264 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚538103092327936 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚533377287618560 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 831258624 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚322 _runtime‚5182.283873473 JDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚65622168 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 122905600 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚464955046985728 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚262 G>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4876 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚45058059 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚20643840 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚41348020 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 5494538240 IBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚60 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚538103151377920 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4289 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 5368709120 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚262 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚ 137990338 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚538103092327936 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 4998233253376 PCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚58387380 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚65617881 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚223 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 122905600 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚469953213459968 M@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚59049984 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚533377270513664ví‡;%¸-ÄO ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 4725821814272 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚72372459 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚61457089 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 4716021943256 F?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚60 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚16398962 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 4628521206 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚72372457 trainer/global_step‚19549 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚4287 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚538103151377920 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚ 137994627 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚72372457 PIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚66 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚534941568568280 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚530225546625024 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 122306526 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 5368709120 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 4654602240 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17104896 KBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚4876 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 125829120 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚45058061 C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚4289 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚469953280239104 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚429 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 4998187118080 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚72372459 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚41945088 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 4654602240 IBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚68 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚61478912 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚322 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚322 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 4552982528 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 4725821814272 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚46135296 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 @7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚5264 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚538103092327936 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚533377287618560 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 831258624 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚322 _runtime‚5182.283873473 JDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚65622168 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 122905600 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚464955046985728 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚262 G>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4876 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚45058059 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚20643840 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚41348020 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 5494538240 IBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚60 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚538103151377920 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4289 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 5368709120 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚262 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚ 137990338 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚538103092327936 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 4998233253376 PCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚58387380 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚65617881 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚223 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 122905600 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚469953213459968 M@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚59049984 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚533377270513664 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 BDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 G@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚68 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚16399798 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚45060264 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚223 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 NADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚58387428 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 4716211167848 trainer/global_step‚19549 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚533403609956352 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚72376012 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚464978073714688 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚429 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚16399732 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚ 138001473 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚4295 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654602240 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 4726094831104 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 I@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚4876 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚533403592851456 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚61478912 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 122306526 _step‚1610 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 4552982528 B9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚5264 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚534967984934476 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521206 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚322 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚209 SFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚66776064 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 845178880 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 4998430379520 K>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59053056 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 4517085184 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚262 G@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚60 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17104896 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 4998476511744 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 4654602240 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚65621168 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚61459996 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚61460064 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚72376010 E5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5264 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚4293 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 831258624 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 125829120 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 4726052882944 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚322 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚538129704787456 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚ 137997178 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚46132224 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 5494538240 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚533403592851456 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚469976570870272 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 125829120 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚45060266 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17104896 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚538129704787456 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚533403609956352 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 122905600 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 138001473 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚262 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚72376010 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚469976504094208 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚322 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚538129645734400 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 5368709120 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚65625461 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 NGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚66 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚464978094358528 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 5494538240 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚322 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚262 % _timestamp‚1.7866638339147787e+09 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 EDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 138001473 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚262 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚72376010 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚469976504094208 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚322 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚538129645734400 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 5368709120 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚65625461 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 NGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚66 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚464978094358528 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 5494538240 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚322 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚262 % _timestamp‚1.7866638339147787e+09 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 EDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 G@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚68 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚16399798 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚45060264 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚223 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 NADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚58387428 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 4716211167848 trainer/global_step‚19549 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚533403609956352 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚72376012 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚464978073714688 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚429 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚16399732 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚ 138001473 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚4295 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654602240 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 4726094831104 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 I@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚4876 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚533403592851456 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚61478912 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 122306526 _step‚1610 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 4552982528 B9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚5264 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚534967984934476 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521206 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚322 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚209 SFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚66776064 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 845178880 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 4998430379520 K>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59053056 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 4517085184 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚262 G@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚60 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17104896 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 4998476511744 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 4654602240 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚65621168 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚61459996 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚61460064 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚72376010 E5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5264 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚4293 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 831258624 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 125829120 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 4726052882944 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚322 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚538129704787456 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚ 137997178 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚46132224Ê FÅ”2¼-- _wandbruntime‚5182  _runtime‚5182´w^œx½-ò &loss/total_loss‚1.7859516231024748 trainer/global_step‚19549 _runtime‚5182.445879119 1loss/FRt1/assignment_loss‚0.45455381697219244 1loss/FRt2/assignment_loss‚0.38009964637374116 0loss/FRt1/detection_loss‚0.46993737139697345 /loss/FRt2/detection_loss‚0.4813607883595676 epoch‚48 % _timestamp‚1.7866638339160204e+09 _step‚1611q{gÅ}¾-÷ 1loss/FRt2/assignment_loss‚0.38009964637374116 0loss/FRt1/detection_loss‚0.46993737139697345 /loss/FRt2/detection_loss‚0.4813607883595676 epoch‚48 % _timestamp‚1.7866638339160204e+09 _step‚1611 &loss/total_loss‚1.7859516231024748 trainer/global_step‚19549 _runtime‚5182.445879119 1loss/FRt1/assignment_loss‚0.45455381697219244Ë ¾V¹¡¿-‚(:– ¡§ùÓÔ‹šá gpu.0.gpu‚62.0 gpu.0.memory‚11, gpu.0.memoryAllocated‚7.0313262939453125) gpu.0.memoryAllocatedBytes‚ 6039863296 gpu.0.temp‚36.0 gpu.0.powerWatts‚226.808' gpu.0.enforcedPowerLimitWatts‚500.0) gpu.0.powerPercent‚45.361599999999996 gpu.0.smClock‚1410 gpu.0.memoryClock‚1593$ gpu.0.correctedMemoryErrors‚1955# gpu.0.uncorrectedMemoryErrors‚0VBÎÀ-‚(:‚ ¡§ùÓйöâ) proc.memory.percent‚14.36917781829834 proc.cpu.threads‚17 network.sent‚ 2.5408745e+07% disk.nvme0c0n1.out‚63461.05859375 disk.sda.in‚ 47844.953125 disk.sda1.in‚0 disk.dm-1.out‚ 95.7734375 disk.sda3.in‚ 47844.953125 disk.sda3.out‚ 95.7734375 disk.sda2.in‚0 disk.sda2.out‚0# disk.nvme0n1.out‚63461.05859375# proc.memory.rssMB‚ 2354.24609375 cpu‚27.260318118776 network.recv‚ 9.705434e+06) disk./.usagePercent‚32.70358953079103$ disk.nvme0c0n1.in‚76263.25390625 disk.sda.out‚ 95.7734375 disk.sda1.out‚0 disk.dm-1.in‚ 47844.953125 disk.drbd2165.in‚0 disk.dm-0.in‚76263.25390625$ disk./.usageGB‚487.3071174621582" disk.nvme0n1.in‚76263.25390625 disk.drbd2165.out‚0 disk.dm-0.out‚63461.05859375$ memory_percent‚68.63811016082764) proc.memory.availableMB‚ 5138.33203125Z!Õ|2Á-- _wandbruntime‚5194  _runtime‚5194vM»ÅÉÂ-à & lr-AdamW/pg1‚1.548667189410935e-06 & lr-AdamW/pg2‚1.548667189410935e-06 trainer/global_step‚19599 % _timestamp‚1.7866638462563994e+09 _runtime‚5194.786422465 _step‚1612i\%›ÎÃ-È & lr-AdamW/pg2‚1.548667189410935e-06 trainer/global_step‚19599 % _timestamp‚1.7866638462563994e+09 _runtime‚5194.786422465 _step‚1612 & lr-AdamW/pg1‚1.548667189410935e-06Ì Æ¸~£2Ä-- _wandbruntime‚5194  _runtime‚5194‰…AÂê$Å-¿O VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚539430855033344 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 4654602240 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚20643840 JDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚534693387403264 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚539430855033344 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚61608321 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 4727583753864 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚41348020 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚41945088 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚61608255 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚466107415625728 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚4289 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 125829120 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚ 138333090 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4289 D;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚5264 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚471117919447552 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚322 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 5368709120 JDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4876 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚65787270 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 5494538240 IBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚60 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚72550109 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 5010459783680 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 125829120 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚539430795983360 BDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚539430795983360 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 4654602240 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0…ù5ëÊ'Æ-ÄO F8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚ 138333090 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4289 D;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚5264 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚471117919447552 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚322 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 5368709120 JDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4876 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚65787270 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 5494538240 IBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚60 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚72550109 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 5010459783680 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 125829120 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚539430795983360 BDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚539430795983360 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 4654602240 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚539430855033344 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 4654602240 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚20643840 JDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚534693387403264 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚539430855033344 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚61608321 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 4727583753864 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚41348020 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚41945088 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚61608255 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚466107415625728 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚4289 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 125829120Í ²Ïçé2Ç-- _wandbruntime‚5194  _runtime‚5194Q Ü8Î&È-ÈM LDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚223 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚536288778034456 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 125829120 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚65786270 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚536288836421884 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 5368709120 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚ 138339930 E5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5264 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚65790563 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚61478912 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚471141143302656 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521206 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 5368709120 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 125829120 $ _timestamp‚1.786663846432715e+09 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚72553662 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚534719726845952 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 4727772978456 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654602240 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 4552982528 NADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚59053056 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 NADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚58387428 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚262 D=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚60 K>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59053056 SFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚68873216 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 845178880 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚ 138344225 E6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚322 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚531561022095360 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 4517085184 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚ 138339930 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚48229376 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚41348068 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 122306526 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚322 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 122905600 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 138344225 BDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚61611251 DDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚471141212175872 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚72553662 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚209 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚322 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚4295 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚539457408442880 P8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 NADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚58387428 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚262 D=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚60 K>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59053056 SFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚68873216 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 845178880 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚ 138344225 E6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚322 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚531561022095360 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 4517085184 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚ 138339930 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚48229376 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚41348068 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 122306526 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚322 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 122905600 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 138344225 BDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚61611251 DDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚471141212175872 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚72553662 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚209 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚322 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚4295 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚539457408442880 PDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 5368709120 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚ 138339930 E5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5264 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚65790563 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚61478912 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚471141143302656 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521206 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 5368709120 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 125829120 $ _timestamp‚1.786663846432715e+09 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚72553662 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚534719726845952 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 4727772978456 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654602240 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 4552982528 NADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚59053056Î 0;«—2Ê-- _wandbruntime‚5194  _runtime‚5194íÿ?&vË-ð _step‚1615 0loss/FRt1/assignment_loss‚0.5455810784412533 epoch‚48 $ _timestamp‚1.786663846433975e+09 1loss/FRt2/assignment_loss‚0.38757114647513846 /loss/FRt1/detection_loss‚0.4864425805235856 0loss/FRt2/detection_loss‚0.45745812349173715 &loss/total_loss‚1.8770529289317144 trainer/global_step‚19599 _runtime‚5194.963811155( Q]{Ì-õ 1loss/FRt2/assignment_loss‚0.38757114647513846 /loss/FRt1/detection_loss‚0.4864425805235856 0loss/FRt2/detection_loss‚0.45745812349173715 &loss/total_loss‚1.8770529289317144 trainer/global_step‚19599 _runtime‚5194.963811155 _step‚1615 0loss/FRt1/assignment_loss‚0.5455810784412533 epoch‚48 $ _timestamp‚1.786663846433975e+09Ï 3÷•”2Í-- _wandbruntime‚5200  _runtime‚5200zþûÎíÎ-Ö 'particle/f_score‚0.5537147521972656 _runtime‚5200.150912354 3Purity/1FRt/event_proportion‚0.6297432780265808 2Purity/2FRt/event_proportion‚0.040062315762043 /particle/accuracy_1_of_2‚0.9248088002204895 (particle/accuracy‚0.6396949291229248 *jet/accuracy_1_of_2‚0.5737868547439575 +particle/sensitivity‚0.6297831535339355 +particle/specificity‚0.6451287865638733 *validation_accuracy‚0.2123507708311081 _step‚1616 2Purity/*FRt/event_proportion‚0.669805645942688 /particle/accuracy_2_of_2‚0.6788069605827332 .Purity/*FRt/FRt_purity‚0.47906503081321716 /Purity/1FRt/event_purity‚0.4678761661052704 /Purity/2FRt/event_purity‚0.4297862648963928 *jet/accuracy_2_of_2‚0.2123507708311081 6validation_average_jet_accuracy‚0.3149946630001068 *jet/accuracy_1_of_1‚0.3100002110004425 )loss/val_total_loss‚2.546554883636701 Purity/0FRt/event_purity‚1 trainer/global_step‚19599 /Purity/*FRt/event_purity‚0.465691715 îË7ï4788971 epoch‚48 % _timestamp‚1.7866638516208754e+09 3Purity/0FRt/event_proportion‚0.3301943838596344 -Purity/1FRt/FRt_purity‚0.4678761661052704 -Purity/2FRt/FRt_purity‚0.5659838318824768 particle/accuracy_1_of_1‚1Ò|’„áÏ-Û *jet/accuracy_1_of_2‚0.5737868547439575 +particle/sensitivity‚0.6297831535339355 +particle/specificity‚0.6451287865638733 *validation_accuracy‚0.2123507708311081 _step‚1616 2Purity/*FRt/event_proportion‚0.669805645942688 /particle/accuracy_2_of_2‚0.6788069605827332 .Purity/*FRt/FRt_purity‚0.47906503081321716 /Purity/1FRt/event_purity‚0.4678761661052704 /Purity/2FRt/event_purity‚0.4297862648963928 *jet/accuracy_2_of_2‚0.2123507708311081 6validation_average_jet_accuracy‚0.3149946630001068 *jet/accuracy_1_of_1‚0.3100002110004425 )loss/val_total_loss‚2.546554883636701 Purity/0FRt/event_purity‚1 trainer/global_step‚19599 /Purity/*FRt/event_purity‚0.4656917154788971 epoch‚48 % _timestamp‚1.7866638516208754e+09 3Purity/0FRt/event_proportion‚0.3301943838596344 -Purity/1FRt/FRt_purity‚0.4678761661052704 -Purity/2FRt/FRt_purity‚0.5659838318824768 particle/accuracy_1_of_1‚1 'particle/f_score‚0.5537147521972656 _runtime‚5200.150912354 3Purity/1FRt/event_proportion‚0.6297432780265808 2Purity/2FRt/event_proportion‚0.040062315762043 /particle/accuracy_1_of_2‚0.9248088002204895 (particle/accuracy‚0.6396949291229248Ð $›œ–Ð-‚(:‹ °§ùÓÝß©á gpu.0.gpu‚64.0 gpu.0.memory‚12, gpu.0.memoryAllocated‚7.0313262939453125) gpu.0.memoryAllocatedBytes‚ 6039863296 gpu.0.temp‚35.0 gpu.0.powerWatts‚216.669' gpu.0.enforcedPowerLimitWatts‚500.0 gpu.0.powerPercent‚43.3338 gpu.0.smClock‚1410 gpu.0.memoryClock‚1593$ gpu.0.correctedMemoryErrors‚1955# gpu.0.uncorrectedMemoryErrors‚0hÛ¥ ‘Ñ-‚(:† °§ùÓÚøÅã' proc.memory.availableMB‚ 5137.921875# proc.memory.rssMB‚ 2354.24609375 proc.cpu.threads‚17 disk.sda3.out‚ 95.953125 disk.sda.out‚ 95.953125 disk.sda1.out‚0 disk.sda2.out‚0# memory_percent‚68.6406135559082) proc.memory.percent‚14.36917781829834 network.sent‚ 2.5483308e+07) disk./.usagePercent‚32.70359746702459% disk./.usageGB‚487.30723571777344# disk.dm-0.out‚63505.28271484375 disk.sda.in‚ 47844.953125& disk.nvme0n1.out‚63505.28271484375 disk.dm-1.out‚ 95.953125 cpu‚27.265146449041318 network.recv‚ 9.736962e+06 disk.dm-0.in‚ 76263.3671875 disk.sda3.in‚ 47844.953125# disk.nvme0c0n1.in‚ 76263.3671875( disk.nvme0c0n1.out‚63505.28271484375 disk.sda1.in‚0 disk.sda2.in‚0 disk.dm-1.in‚ 47844.953125 disk.drbd2165.in‚0 disk.drbd2165.out‚0! disk.nvme0n1.in‚ 76263.3671875Ô"¦2Ò-- _wandbruntime‚5212  _runtime‚5212ÉpéÚËÓ-Å trainer/global_step‚19649 % _timestamp‚1.7866638644692285e+09 _runtime‚5212.999286867 _step‚1617 ' lr-AdamW/pg1‚1.1866388087256596e-06 ' lr-AdamW/pg2‚1.1866388087256596e-06û*>DÐÔ-Ê ' lr-AdamW/pg2‚1.1866388087256596e-06 trainer/global_step‚19649 % _timestamp‚1.7866638644692285e+09 _runtime‚5212.999286867 _step‚1617 ' lr-AdamW/pg1‚1.1866388087256596e-06Ñ ÿè¬2Õ-- _wandbruntime‚5213  _runtime‚5213mÕÅ'Ö-¿O [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 4753105014784 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 122905600 D;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚5264 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚16489218 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚472585963258880 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚20643840 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚61478912 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚41945088 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17104896 JBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚665 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚65974559 PIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚65 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚16489153 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚467559033929728 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 4517085184 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚41348020 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 125829120 C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚4289 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚65970272 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 5026908685312 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚541153563627520 F>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚262 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚4287 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚44038144 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚536400458612736 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚537968237007037 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚533225008431104 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 KDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚60 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚61808393 PCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚58387380 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚72788363 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚536400441507840 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 4654602240 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 4552982528 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚65970272 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚209 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 5494538240 @7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚5264 IBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚67 BDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚541153504577536 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 4628521206 _runtime‚5213.003933667 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚65974559 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 4753063069696 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚467559054573568 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚533225025470464 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚41945088 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 122306526 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 5368709120 JDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4876 UHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚64681984 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 125829120 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 4654602240 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚223 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 831258624 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17104896 JDeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4289 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 5368709120 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 4552982528 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 4753105014784 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚72788365 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚45319173 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 845178880 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚262 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚72788363 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚537968178619657 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 125829120 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚262 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 4753063069696 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 5494538240 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚322 M@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚59049984 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 5494538240 IBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚60 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚541153563627520 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚ 138762924 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚429 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚541153504577536 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 4743170188553 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚322 % _timestamp‚1.7866638644702795e+09 _step‚1618 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚322 F?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚60 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚ 138758635 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚72788365 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚4287 trainer/global_step‚19649LÙ[tÊ'×-ÄO JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚4287 trainer/global_step‚19649 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 4753105014784 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 122905600 D;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚5264 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚16489218 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚472585963258880 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚20643840 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚61478912 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚41945088 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17104896 JBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚665 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚65974559 PIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚65 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚16489153 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚467559033929728 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 4517085184 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚41348020 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 125829120 C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚4289 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚65970272 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 5026908685312 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚541153563627520 F>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚262 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚4287 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚44038144 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚536400458612736 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚537968237007037 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚533225008431104 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 KDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚60 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚61808393 PCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚58387380 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚72788363 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚536400441507840 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 4654602240 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 4552982528 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚65970272 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚209 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 5494538240 @7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚5264 IBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚67 BDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚541153504577536 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 4628521206 _runtime‚5213.003933667 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚65974559 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 4753063069696 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚467559054573568 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚533225025470464 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚41945088 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 122306526 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 5368709120 JDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4876 UHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚64681984 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 125829120 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 4654602240 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚223 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 831258624 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17104896 JDeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4289 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 5368709120 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 4552982528 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 4753105014784 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚72788365 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚45319173 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 845178880 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚262 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚72788363 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚537968178619657 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 125829120 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚262 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 4753063069696 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 5494538240 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚322 M@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚59049984 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 5494538240 IBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚60 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚541153563627520 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚ 138762924 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚429 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚541153504577536 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 4743170188553 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚322 % _timestamp‚1.7866638644702795e+09 _step‚1618 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚322 F?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚60 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚ 138758635 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚72788365Ò }àc2Ø-- _wandbruntime‚5213  _runtime‚5213'¿—’õ"Ù-ÉM REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚41948160 NGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚73 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 125829120 DDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 4743400761165 G@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚60 PDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚61811326 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚16489948 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚472609184328704 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚52423680 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 125829120 NADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚59053056 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654602240 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚536426780950528 B9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚5264 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚45321380 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚61478912 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚533251211264000 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚65973559 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚541180057984000 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚65977852 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚41348068 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 IBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚60 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚4295 K>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59053056 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 5027103026176 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 5494538240 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 5494538240 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚322 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚322 D=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚60 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚65973559 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 138769770 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚72791918 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚ 138769770 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 122905600 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚45321378 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚537994653373233 % _timestamp‚1.7866638646465528e+09 >5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5264 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚322 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚209 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚467582101946368 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚20643840 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚467582081302528 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 4517085184 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚262 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 4753336086528 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 4552982528 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 5368709120 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 122905600 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚262 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚536426763845632 EDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚4293 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 4753336086528 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚72791916 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 4753294138368 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚61811401 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 5368709120 _runtime‚5213.177025187 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚223 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 5027155449856 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 4552982528 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521206 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 4654602240 >8DeviceStatsMonitor.on_train_batc$_¢ÄÚh_end/num_oom_rejections‚0 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚536426763845632 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚41948160 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚16490021 SFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚73067520 _step‚1619 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚429 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 845178880 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 4743442109233 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 125829120 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚262 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17104896 trainer/global_step‚19649ƒ6•EÔ&Ú-ÎM LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚209 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚467582101946368 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚20643840 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚467582081302528 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 4517085184 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚262 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 4753336086528 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 4552982528 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 5368709120 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 122905600 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚262 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚536426763845632 EDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚4293 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 4753336086528 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚72791916 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 4753294138368 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚61811401 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 5368709120 _runtime‚5213.177025187 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚223 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 5027155449856 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 4552982528 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521206 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 4654602240 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚536426763845632 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚41948160 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚16490021 SFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚73067520 _step‚1619 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚429 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 845178880 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 4743442109233 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 125829120 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚262 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17104896 trainer/global_step‚19649 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚41948160 NGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚73 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 125829120 DDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 4743400761165 G@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚60 PDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚61811326 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚16489948 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚472609184328704 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚52423680 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 125829120 NADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚59053056 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654602240 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚536426780950528 B9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚5264 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚45321380 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚61478912 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚533251211264000 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚65973559 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚541180057984000 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚65977852 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚41348068 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 IBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚60 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚4295 K>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59053056 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 5027103026176 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 5494538240 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 5494538240 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚322 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚322 D=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚60 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚65973559 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 138769770 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚72791918 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚ 138769770 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 122905600 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚45321378 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚537994653373233 % _timestamp‚1.7866638646465528e+09 >5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5264 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚322Ó ky)2Û-- _wandbruntime‚5213  _runtime‚5213·ò þuÜ-ï 0loss/FRt1/assignment_loss‚0.4243746485070138 /loss/FRt2/assignment_loss‚0.392826278505956 /loss/FRt1/detection_loss‚0.4872503841111103 0loss/FRt2/detection_loss‚0.45517247605642874 &loss/total_loss‚1.7596237871805087 epoch‚49 _runtime‚5213.177727053 _step‚1620 trainer/global_step‚19649 % _timestamp‚1.7866638646478343e+09õrçzÝ-ô epoch‚49 _runtime‚5213.177727053 _step‚1620 trainer/global_step‚19649 % _timestamp‚1.7866638646478343e+09 0loss/FRt1/assignment_loss‚0.4243746485070138 /loss/FRt2/assignment_loss‚0.392826278505956 /loss/FRt1/detection_loss‚0.4872503841111103 0loss/FRt2/detection_loss‚0.45517247605642874 &loss/total_loss‚1.7596237871805087Ô 5?E–Þ-‚(:‹ ¿§ùÓ…‘ôà gpu.0.gpu‚60.0 gpu.0.memory‚11, gpu.0.memoryAllocated‚7.0313262939453125) gpu.0.memoryAllocatedBytes‚ 6039863296 gpu.0.temp‚37.0 gpu.0.powerWatts‚223.318' gpu.0.enforcedPowerLimitWatts‚500.0 gpu.0.powerPercent‚44.6636 gpu.0.smClock‚1410 gpu.0.memoryClock‚1593$ gpu.0.correctedMemoryErrors‚1955# gpu.0.uncorrectedMemoryErrors‚0B¹‡ž–ß-‚(:‹ ¿§ùÓìò¹ã disk.sda.out‚ 96.08203125 disk.sda2.in‚0 disk.nvme0n1.in‚ 76269.796875& disk.nvme0n1.out‚63513.31103515625 disk.drbd2165.in‚0 disk.drbd2165.out‚0 disk.dm-0.in‚ 76269.796875 disk.sda3.in‚ 47844.953125 network.recv‚ 9.757386e+06 disk.sda.in‚ 47844.953125 disk.sda1.in‚0 disk.sda1.out‚0 disk.sda2.out‚0# disk.dm-0.out‚63513.31103515625 disk.sda3.out‚ 96.08203125( disk.nvme0c0n1.out‚63513.31103515625) disk./.usagePercent‚32.70371676653554 disk.dm-1.in‚ 47844.953125" disk.nvme0c0n1.in‚ 76269.796875) proc.memory.availableMB‚ 5137.98828125# proc.memory.rssMB‚ 2354.24609375 proc.cpu.threads‚17 network.sent‚ 2.5548191e+07$ disk./.usageGB‚487.3090133666992 disk.dm-1.out‚ 96.08203125$ memory_percent‚68.64020824432373) proc.memory.percent‚14.36917781829834 cpu‚27.264771402072093eúÅ2à-- _wandbruntime‚5225  _runtime‚5225&Ó“ËÉá-à _runtime‚5225.463414876 _step‚1621 & lr-AdamW/pg1‚8.727053109577887e-07 & lr-AdamW/pg2‚8.727053109577887e-07 trainer/global_step‚19699 % _timestamp‚1.7866638769333978e+09ùôÁrÎâ-È _runtime‚5225.463414876 _step‚1621 & lr-AdamW/pg1‚8.727053109577887e-07 & lr-AdamW/pg2‚8.727053109577887e-07 trainer/global_step‚19699 % _timestamp‚1.7866638769333978e+09Õ \Ç2ã-- _wandbruntime‚5225  _runtime‚5225Wí¡ÑÅ'ä-¿O L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚72966013 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 122905600 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚ 139105676 IBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚79 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 4628521206 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚262 IBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚60 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚537716575502336 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚473750534841856 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚468711402569728 JDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚542481208232960 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚534534315147264 _runtime‚5225.467873383 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 125829120 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚542481267282944 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚45429423 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚16530272 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚473750605815296 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚ 139101387 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 4764691780608 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 4754773347181 BDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4876 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚66139661 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 4517085184 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 5494538240 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 122905600 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚72966015 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚539289088494445 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚262 D;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚5264 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 122306526 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚41945088 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 4552982528 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚322 UHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚70973440 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚322 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 4764649835520 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17104896 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 F?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚60 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚16530349 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 845178880 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 125829120 KDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚60 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17104896 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 4764691780608 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 125829120 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚262 KBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚4876 M@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚59049984 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 5039132272128 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 4754731999161 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 PIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚77 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 trainer/global_step‚19699 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚72966015 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚4287 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚534534298107904 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚41348020 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚537716558397440 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚542481208232960 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚429 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 831258624 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚66135374 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚223 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚209 JBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚665 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 5368709120 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 4654602240 F>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚4287 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 4552982528 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚61959695 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚50329600 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 5494538240 PCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚59049984 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 4764649835520 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚45429425 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 5368709120 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚537716575502336 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚468711423213568 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚322 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚542481267282944 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 4654602240 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚537716558397440 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4289 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚41945088 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚72966013 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 PCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚58387380 % _timestamp‚1.7866638769343188e+09 _step‚1622 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚61478912 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 5368709120÷}Ìê"å-ÄO F8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚ 139101387 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 4764691780608 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 4754773347181 BDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4876 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚66139661 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 4517085184 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 5494538240 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 122905600 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚72966015 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚539289088494445 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚262 D;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚5264 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 122306526 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚41945088 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 4552982528 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚322 UHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚70973440 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚322 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 4764649835520 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17104896 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 F?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚60 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚16530349 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 845178880 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 125829120 KDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚60 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17104896 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 4764691780608 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 125829120 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚262 KBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚4876 M@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚59049984 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 5039132272128 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 4754731999161 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 PIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚77 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 trainer/global_step‚19699 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚72966015 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚4287 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚534534298107904 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚41348020 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚537716558397440 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚542481208232960 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚429 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 831258624 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚66135374 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚223 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚209 JBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚665 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 5368709120 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 4654602240 F>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚4287 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 4552982528 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚61959695 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚50329600 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 5494538240 PCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚59049984 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 4764649835520 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚45429425 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 5368709120 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚537716575502336 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚468711423213568 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚322 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚542481267282944 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 4654602240 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚537716558397440 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4289 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚41945088 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚72966013 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 PCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚58387380 % _timestamp‚1.7866638769343188e+09 _step‚1622 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚61478912 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 5368709120 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚72966013 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 122905600 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚ 139105676 IBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚79 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 4628521206 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚262 IBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚60 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚537716575502336 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚473750534841856 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚468711402569728 JDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚542481208232960 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚534534315147264 _runtime‚5225.467873383 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 125829120 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚542481267282944 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚45429423 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚16530272 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚473750605815296Ö ÷2æ-- _wandbruntime‚5225  _runtime‚5225Úc'*Î&ç-ÈM BDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654602240 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚66138661 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 122306526 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚542507820692480 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 4552982528 B9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚5264 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 831258624 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚539315504860641 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚322 E5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5264 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚72969566 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚72969568 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 5039425532416 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 125829120 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚262 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17104896 NADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚59053056 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚322 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 4764880904192 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚ 139108227 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚468734449942528 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚262 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚4293 I@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚4876 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚16531121 NGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚68 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 845178880 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 5368709120 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚322 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚ 139112522 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚46132224 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 DDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59053056 D=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚60 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚4295 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚66142954 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚534560500940800 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 125829120 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚45431630 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 5494538240 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚537742897840128 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 125829120 _step‚1623 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚ 139108227 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚542507761639424 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 4764922852352 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚468734470586368 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚537742880735232 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚66142954 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚473773829342720 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 trainer/global_step‚19699 P8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚534560483901440 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚262 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 122905600 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 4754962571773 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 5368709120 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚72969566 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 4654602240 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 4552982528 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17104896 EDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521206 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚61962751 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 4764922852352 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚61962681 SFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚66776064 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚20643840 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚61478912 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚41348068 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚4293 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚16531053 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 139112522 $ _timestamp‚1.786663877097976e+09 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 4764880904192 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 5494538240 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚429 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚537742880735232 IBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚60 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚72969568 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 122905600 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚45431628 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚322 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚542507820692480 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 4517085184 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 4755003919841 G@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚60 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚473773896118784ÕÉPÓ&è-ÍM XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 4754962571773 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 5368709120 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚72969566 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 4654602240 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 4552982528 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17104896 EDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521206 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚61962751 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 4764922852352 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚61962681 SFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚66776064 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚20643840 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚61478912 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚41348068 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚4293 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚16531053 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 139112522 $ _timestamp‚1.786663877097976e+09 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 4764880904192 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 5494538240 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚429 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚537742880735232 IBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚60 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚72969568 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 122905600 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚45431628 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚322 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚542507820692480 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 4517085184 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 4755003919841 G@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚60 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚473773896118784 BDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654602240 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚66138661 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 122306526 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚542507820692480 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 4552982528 B9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚5264 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 831258624 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚539315504860641 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚322 E5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5264 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚72969566 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚72969568 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 5039425532416 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 125829120 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚262 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17104896 NADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚59053056 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚322 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 4764880904192 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚ 139108227 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚468734449942528 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚262 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚4293 I@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚4876 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚16531121 NGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚68 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 845178880 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 5368709120 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚322 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚ 139112522 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚46132224 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 DDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59053056 D=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚60 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚4295 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚66142954 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚534560500940800 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 125829120 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚45431630 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 5494538240 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚537742897840128 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 125829120 _step‚1623 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚ 139108227 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚542507761639424 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 4764922852352 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚468734470586368 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚537742880735232 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚66142954 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚473773829342720 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 trainer/global_step‚19699 P8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚534560483901440 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚262 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 122905600 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0× h¸z£2é-- _wandbruntime‚5225  _runtime‚5225V€Švê-ð 0loss/FRt1/assignment_loss‚0.4697736700015924 1loss/FRt2/assignment_loss‚0.36626602429924887 &loss/total_loss‚1.8240588428851456 epoch‚49 trainer/global_step‚19699 _runtime‚5225.628791501 /loss/FRt1/detection_loss‚0.4956444087684076 /loss/FRt2/detection_loss‚0.4923747398158967 % _timestamp‚1.7866638770989327e+09 _step‚16244ý¤A{ë-õ trainer/global_step‚19699 _runtime‚5225.628791501 /loss/FRt1/detection_loss‚0.4956444087684076 /loss/FRt2/detection_loss‚0.4923747398158967 % _timestamp‚1.7866638770989327e+09 _step‚1624 0loss/FRt1/assignment_loss‚0.4697736700015924 1loss/FRt2/assignment_loss‚0.36626602429924887 &loss/total_loss‚1.8240588428851456 epoch‚49Ø í á8–ì-‚(:‹ ΧùÓÿë’á gpu.0.gpu‚56.0 gpu.0.memory‚11, gpu.0.memoryAllocated‚7.0313262939453125) gpu.0.memoryAllocatedBytes‚ 6039863296 gpu.0.temp‚35.0 gpu.0.powerWatts‚123.614' gpu.0.enforcedPowerLimitWatts‚500.0 gpu.0.powerPercent‚24.7228 gpu.0.smClock‚1410 gpu.0.memoryClock‚1593$ gpu.0.correctedMemoryErrors‚1955# gpu.0.uncorrectedMemoryErrors‚0tŽ9ùŠí-‚(:ÿ ΧùÓâÖšã disk.sda3.out‚ 96.3515625 disk.sda2.out‚0$ memory_percent‚68.64030361175537) proc.memory.availableMB‚ 5137.97265625 proc.cpu.threads‚17 network.sent‚ 2.5623718e+07$ disk./.usageGB‚487.4339027404785 disk.dm-1.out‚ 96.3515625 disk.dm-0.out‚63557.31640625" disk.nvme0c0n1.in‚ 76269.796875% disk.nvme0c0n1.out‚63557.31640625 disk.sda1.out‚0# proc.memory.rssMB‚ 2354.24609375 network.recv‚ 9.790135e+06) disk./.usagePercent‚32.71209819719904# disk.nvme0n1.out‚63557.31640625 disk.dm-1.in‚ 47844.953125 disk.sda.in‚ 47844.953125 disk.sda.out‚ 96.3515625 disk.sda2.in‚0) proc.memory.percent‚14.36917781829834 disk.nvme0n1.in‚ 76269.796875 disk.drbd2165.out‚0 disk.dm-0.in‚ 76269.796875 disk.sda3.in‚ 47844.953125 disk.sda1.in‚0 cpu‚27.264210274331376 disk.drbd2165.in‚0\±1÷2î-- _wandbruntime‚5238  _runtime‚5238%ÔUÉï-à _step‚1625 & lr-AdamW/pg1‚6.068868594693022e-07 & lr-AdamW/pg2‚6.068868594693022e-07 trainer/global_step‚19749 % _timestamp‚1.7866638896872816e+09 _runtime‚5238.217333973«‹÷Îð-È & lr-AdamW/pg1‚6.068868594693022e-07 & lr-AdamW/pg2‚6.068868594693022e-07 trainer/global_step‚19749 % _timestamp‚1.7866638896872816e+09 _runtime‚5238.217333973 _step‚1625Ù ÄÁÔ2ñ-- _wandbruntime‚5238  _runtime‚5238ú÷­"ò-¿O GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4289 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚73143663 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚469863771209728 C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚4289 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚474915270426624 KDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚60 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚66304763 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 4552982528 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚62110867 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚535843587784704 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 4766335157789 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚539032675287040 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 4776278546432 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚4287 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚16571192 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 PCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚58387380 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚540609881594473 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 5494538240 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17104896 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚543808970938368 JBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚665 UHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚68876288 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 4776236601344 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚66300476 D;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚5264 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 5368709120 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 4552982528 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚535843604824064 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 4517085184 % _timestamp‚1.7866638896882439e+09 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚73143665 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚41348020 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚73143663 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 5368709120 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚61478912 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 4654602240 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 122905600 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚539032675287040 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚469863791853568 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚48232448 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 5494538240 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 4776236601344 PCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚59049984 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 4766293809769 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚543808911888384 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚474915201550336 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚543808970938368 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚322 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 5368709120 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚262 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚262 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚539032692391936 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚429 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 845178880 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚20643840 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 _runtime‚5238.218508563 @7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚5264 G>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4876 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 BDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚543808911888384 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚223 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 `LDeviceStatsMonitor.on_train_batch_star)ºt/allocated_bytes.large_pool.allocated‚539032692391936 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 125829120 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚66300476 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 5051430340608 F>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚45539673 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚16571115 JDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚45539673 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚16571115 JDeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4289 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚73143663 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚469863771209728 C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚4289 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚474915270426624 KDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚60 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚66304763 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 4552982528 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚62110867 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚535843587784704 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 4766335157789 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚539032675287040 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 4776278546432 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚4287 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚16571192 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 PCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚58387380 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚540609881594473 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 5494538240 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17104896 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚543808970938368 JBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚665 UHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚68876288 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 4776236601344 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚66300476 D;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚5264 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 5368709120 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 4552982528 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚535843604824064 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 4517085184 % _timestamp‚1.7866638896882439e+09 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚73143665 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚41348020 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚73143663 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 5368709120 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚61478912 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 4654602240 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 122905600 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚539032675287040 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚469863791853568 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚48232448 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 5494538240 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 4776236601344 PCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚59049984 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 4766293809769 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚543808911888384 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚474915201550336 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚543808970938368 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚322 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 5368709120 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚262 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚262 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚539032692391936 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚429 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 845178880 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚20643840 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 _runtime‚5238.218508563 @7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚5264 G>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4876 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 BDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚543808911888384 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚223 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚539032692391936 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 125829120Ú XX¨ 2ô-- _wandbruntime‚5238  _runtime‚5238wB—Í&õ-ÇM >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚322 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚73147218 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚474938563589120 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 125829120 EDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 125829120 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 125829120 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚543835465294848 H@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚665 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚262 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 122905600 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 139455274 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚4293 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚16571991 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 G@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚60 _step‚1627 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 4552982528 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 122905600 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚62113792 NGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚77 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚535869790617600 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 5494538240 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚66308056 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚41948160 G@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚79 NADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚58387428 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚429 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚41348068 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚ 139450979 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚66303763 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚209 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 4517085184 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚4293 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚50326528 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚41948160 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚469886818582528 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 4766524382381 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 5494538240 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 5494538240 D=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚60 >5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5264 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚539058997624832 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 4776467670016 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 _runtime‚5238.384957772 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚543835524347904 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17104896 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 5051674036224 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521206 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚62113871 D6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚322 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 4776467670016 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚535869773578240 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚539058997624832 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 831258624 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚540636356348049 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 5368709120 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 4776509618176 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚539059014729728 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚73147216 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚73147216 SFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚70970368 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚474938492618752 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚322 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 4654602240 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 5051724362752 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 4766565730449 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 5368709120 IBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚60 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚ 139450979 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚543835524347904 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚66308056 I@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚4876 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚45541880 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚322 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚539059014729728 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚61478912 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 5368709120 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚73147218 EDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654602240 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 4776509618176 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚223 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚469886839226368 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚262 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚4295 K>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59053056 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚45541878 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚262 # _timestamp‚1.78666388985445e+09 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚66303763 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 122306526½[ +Ò&ö-ÌM EDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654602240 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 4776509618176 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚223 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚469886839226368 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚262 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚4295 K>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59053056 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚45541878 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚262 # _timestamp‚1.78666388985445e+09 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚66303763 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 122306526 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚322 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚73147218 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚474938563589120 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 125829120 EDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 125829120 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 125829120 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚543835465294848 H@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚665 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚262 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 122905600 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 139455274 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚4293 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚16571991 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 G@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚60 _step‚1627 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 4552982528 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 122905600 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚62113792 NGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚77 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚535869790617600 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 5494538240 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚66308056 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚41948160 G@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚79 NADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚58387428 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚429 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚41348068 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚ 139450979 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚66303763 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚209 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 4517085184 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚4293 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚50326528 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚41948160 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚469886818582528 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 4766524382381 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 5494538240 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 5494538240 D=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚60 >5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5264 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚539058997624832 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 4776467670016 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 _runtime‚5238.384957772 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚543835524347904 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17104896 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 5051674036224 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521206 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚62113871 D6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚322 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 4776467670016 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚535869773578240 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚539058997624832 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 831258624 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚540636356348049 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 5368709120 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 4776509618176 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚539059014729728 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚73147216 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚73147216 SFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚70970368 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚474938492618752 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚322 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 4654602240 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 5051724362752 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 4766565730449 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 5368709120 IBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚60 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚ 139450979 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚543835524347904 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚66308056 I@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚4876 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚45541880 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚322 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚539059014729728 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚61478912 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 5368709120 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚73147218Û ,/1A2÷-- _wandbruntime‚5238  _runtime‚5238l¨5vø-ð trainer/global_step‚19749 % _timestamp‚1.7866638898554332e+09 _step‚1628 0loss/FRt2/assignment_loss‚0.3731613700207351 /loss/FRt1/detection_loss‚0.4746470030867085 0loss/FRt2/detection_loss‚0.46607880614746794 _runtime‚5238.385451245 1loss/FRt1/assignment_loss‚0.49720697593522256 %loss/total_loss‚1.811094155190134 epoch‚49ïÏ{ù-õ 1loss/FRt1/assignment_loss‚0.49720697593522256 %loss/total_loss‚1.811094155190134 epoch‚49 trainer/global_step‚19749 % _timestamp‚1.7866638898554332e+09 _step‚1628 0loss/FRt2/assignment_loss‚0.3731613700207351 /loss/FRt1/detection_loss‚0.4746470030867085 0loss/FRt2/detection_loss‚0.46607880614746794 _runtime‚5238.385451245Ü š–ú-‚(:‹ ݧùӲ߉á gpu.0.gpu‚58.0 gpu.0.memory‚10, gpu.0.memoryAllocated‚7.0313262939453125) gpu.0.memoryAllocatedBytes‚ 6039863296 gpu.0.temp‚35.0 gpu.0.powerWatts‚216.883' gpu.0.enforcedPowerLimitWatts‚500.0 gpu.0.powerPercent‚43.3766 gpu.0.smClock‚1410 gpu.0.memoryClock‚1593$ gpu.0.correctedMemoryErrors‚1955# gpu.0.uncorrectedMemoryErrors‚0âÙ%Nû-‚(:‚ ݧùÓœ´ýã disk.drbd2165.out‚0 disk.sda3.out‚ 96.480467R.R?875% disk.nvme0c0n1.out‚63565.34765625 disk.sda1.in‚0) proc.memory.percent‚14.36917781829834) disk./.usagePercent‚32.71209845320657$ disk./.usageGB‚487.4339065551758 disk.nvme0n1.in‚ 76269.796875# disk.nvme0n1.out‚63565.34765625 disk.dm-0.out‚63565.34765625 disk.sda1.out‚0 disk.sda2.out‚0 proc.cpu.threads‚17 network.sent‚ 2.5698608e+07 disk.sda3.in‚ 47844.953125 disk.sda.in‚ 47844.953125 disk.sda2.in‚0) proc.memory.availableMB‚ 5137.96484375# proc.memory.rssMB‚ 2354.24609375 cpu‚27.263688897348075 disk.dm-1.in‚ 47844.953125 disk.dm-1.out‚ 96.48046875 disk.drbd2165.in‚0 disk.dm-0.in‚ 76269.796875" disk.nvme0c0n1.in‚ 76269.796875 disk.sda.out‚ 96.48046875$ memory_percent‚68.64035129547119 network.recv‚ 9.822556e+06†É(™2ü-- _wandbruntime‚5250  _runtime‚5250¥¯>»Êý-Ä trainer/global_step‚19799 $ _timestamp‚1.786663902205235e+09 _runtime‚5250.735248887 _step‚1629 ' lr-AdamW/pg1‚3.8920052728261267e-07 ' lr-AdamW/pg2‚3.8920052728261267e-07g#:Ïþ-É ' lr-AdamW/pg1‚3.8920052728261267e-07 ' lr-AdamW/pg2‚3.8920052728261267e-07 trainer/global_step‚19799 $ _timestamp‚1.786663902205235e+09 _runtime‚5250.735248887 _step‚1629Ý ò¾±Ó2ÿ-- _wandbruntime‚5250  _runtime‚5250V'æÄ'€.¾O WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 125829120 IBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚60 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚66469865 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 5063717399552 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚541930733081881 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚476079857249280 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚46135296 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 5494538240 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚322 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚545136615543808 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚16612456 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 125829120 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 122905600 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚66465578 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚61478912 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 4654602240 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 4787865312256 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 4628521206 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚537152877461504 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚ 139786891 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17104896 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚209 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 JDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚545136615543808 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 4552982528 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚262 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚73321313 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚16612379 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 _step‚1630 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚545136674593792 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚471016160493568 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚540348792176640 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 4552982528 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚541930791469261 PCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚58387380 @7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚5264 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 KBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚4876 UHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚66779136 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 4517085184 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚322 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 4787823367168 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚429 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 BDeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4289 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚73321315 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚62262302 F?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚60 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 4654602240 JDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4876 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚545136674593792 D;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚5264 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚45649925 trainer/global_step‚19799 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚ 139791180 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚73321315 JBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚665 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 125829120 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚41945088 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚62262381 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 M@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚59049984 IBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚79 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 4777855620377 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚262 PCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚59049984 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 5063763534848 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 5494538240 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 4777896968397 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 4787865312256 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 831258624 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚537152894500864 F>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚66465578 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚41945088 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 4787823367168 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 122306526 _runtime‚5250.736422045 C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚4289 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚66469865 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚471016139849728 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚41348020 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 5368709120 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚322 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚540348792176640 KDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚60 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚73321313 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 122905600 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚45649923 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 5368709120 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 $ _timestamp‚1.786663902206178e+09 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚4287 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚223 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 845178880Ž\@É'.ÃO \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚20643840 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚540348809281536 PIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚77 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚322 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚545136615543808 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 4552982528 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚262 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚73321313 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚16612379 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 _step‚1630 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚545136674593792 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚471016160493568 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚540348792176640 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 4552982528 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚541930791469261 PCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚58387380 @7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚5264 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 KBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚4876 UHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚66779136 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 4517085184 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚322 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 4787823367168 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚429 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 BDeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4289 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚73321315 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚62262302 F?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚60 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 4654602240 JDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4876 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚545136674593792 D;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚5264 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚45649925 trainer/global_step‚19799 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚ 139791180 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚73321315 JBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚665 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 125829120 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚41945088 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚62262381 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 M@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚59049984 IBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚79 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 4777855620377 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚262 PCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚59049984 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 5063763534848 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 5494538240 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 4777896968397 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 4787865312256 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 831258624 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚537152894500864 F>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚66465578 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚41945088 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 4787823367168 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 122306526 _runtime‚5250.736422045 C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚4289 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚66469865 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚471016139849728 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚41348020 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 5368709120 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚322 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚540348792176640 KDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚60 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚73321313 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 122905600 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚45649923 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 5368709120 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 $ _timestamp‚1.786663902206178e+09 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚4287 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚223 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 845178880 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 125829120 IBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚60 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚66469865 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 5063717399552 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚541930733081881 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚476079857249280 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚46135296 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 5494538240 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚322 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚545136615543808 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚16612456 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 125829120 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 122905600 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚66465578 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚61478912 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 4654602240 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 4787865312256 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 4628521206 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚537152877461504 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚ 139786891 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17104896 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚209 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654602240 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 4552982528 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚223 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚61478912 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17104896 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 122905600 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚16613221 SFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚66776064 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚537179063255040 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 4788096384000 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚73324866 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚476103150177280 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚540375131619328 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚471039207866368 H@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚665 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521206 DDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 139798026 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚66473158 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚429 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 5063962954752 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚41348068 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 125829120 D=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚60 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚66468865 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚16613286 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 5368709120 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚322 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚73324868 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚541957149448029 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 5494538240 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚41948160 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚545163228003328 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 K>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59053056 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 4552982528 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 4788054435840 G@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚67 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 5064009086976 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚262 IBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚60 % _timestamp‚1.7866639023686907e+09 >5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5264 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚322 _runtime‚5250.899032626 NADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚59053056 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚545163168950272 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 I@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚4876 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 125829120 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚4293 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 5494538240 _step‚1631 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 4654602240 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚41948160 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚45652128 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 845178880 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚540375114514432 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚209 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 4517085184 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 4778127541057 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚322 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 122306526 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚262 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 NADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚58387428 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 5368709120 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚66473158 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17104896 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚ 139793731 B9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚5264 NGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚65 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚322 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 5368709120 trainer/global_step‚19799 EDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚62265349 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚20643840 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚541957207835457 EDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654602240 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 4552982528 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚223 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚61478912 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17104896 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 122905600 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚16613221 SFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚66776064 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚537179063255040 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 4788096384000 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚73324866 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚476103150177280 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚540375131619328 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚471039207866368 H@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚665 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521206 DDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 139798026 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚66473158 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚429 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 5063962954752 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚41348068 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 125829120 D=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚60 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚66468865 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚16613286 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 5368709120 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚322 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚73324868 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚541957149448029 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 5494538240 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚41948160 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚545163228003328 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 K>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59053056 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 4552982528 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 4788054435840 G@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚67 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 5064009086976 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚262 IBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚60 % _timestamp‚1.7866639023686907e+09 >5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5264 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚322 _runtime‚5250.899032626 NADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚59053056 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚545163168950272 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 I@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚4876 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 125829120 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚4293 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 5494538240 _step‚1631 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 4654602240 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚41948160 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚45652128 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 845178880 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚540375114514432 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚209 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 4517085184 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 4778127541057 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚322 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 122306526 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚262 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 NADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚58387428 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 5368709120 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚66473158 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17104896 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚ 139793731 B9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚5264 NGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚65 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚322 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 5368709120 trainer/global_step‚19799 EDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚62265349 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚20643840 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚541957207835457 EDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4876 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚73498965 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚73498963 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚ 140129643 PCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚59049984 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17104896 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 4799410132992 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚44038144 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚41348020 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 125829120 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 % _timestamp‚1.7866639151742513e+09 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 4552982528 IBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚69 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚4287 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 5076039153152 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚61478912 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 4628521206 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 4789458779005 M@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚59049984 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 4552982528 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚41945088 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚41945088 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚477244568286720 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚546464378249216 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚543251584569289 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚322 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4289 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 4517085184 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚66630680 PIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚67 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17104896 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚209 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚472168529133568 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚538462167138304 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 _runtime‚ 5263.70787716 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚546464378249216 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 5494538240 _step‚1634 C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚4289 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 5368709120 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚538462184177664 @7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚5264 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 5494538240 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚429 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚45760173 PCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚58387380 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚66630680 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚62413246 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 831258624 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 5075995115008 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚66634967 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚16653140 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚20643840 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚262 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚541664909066240 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚62413315 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚477244503604736 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚262 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 125829120 D;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚5264 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 845178880 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚543251642956669 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 122306526 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 125829120 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚322 F>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 JBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚665 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 KDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚60 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚ 140133932 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚472168508489728 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 4799410132992 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚541664926171136 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚322 trainer/global_step‚19849 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚66634967 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚546464319199232 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 122905600 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 5368709120 IBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚60ÊÅå¿É'.ÃO C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚4289 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 5368709120 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚538462184177664 @7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚5264 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 5494538240 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚429 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚45760173 PCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚58387380 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚66630680 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚62413246 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 831258624 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 5075995115008 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚66634967 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚16653140 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚20643840 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚262 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚541664909066240 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚62413315 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚477244503604736 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚262 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 125829120 D;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚5264 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 845178880 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚543251642956669 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 122306526 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 125829120 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚322 F>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 JBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚665 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 KDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚60 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚ 140133932 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚472168508489728 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 4799410132992 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚541664926171136 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚322 trainer/global_step‚19849 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚66634967 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚546464319199232 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 122905600 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 5368709120 IBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚60 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚546464319199232 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚73498963 KBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚4876 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚45760175 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 5494538240 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 4799452078080 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 4654602240 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚4287 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚16653073 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚322 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 4789417430985 JDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4876 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚73498965 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚73498963 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚ 140129643 PCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚59049984 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17104896 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 4799410132992 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚44038144 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚41348020 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 125829120 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 % _timestamp‚1.7866639151742513e+09 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 4552982528 IBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚69 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚4287 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 5076039153152 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚61478912 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 4628521206 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 4789458779005 M@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚59049984 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 4552982528 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚41945088 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚41945088 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚477244568286720 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚546464378249216 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚543251584569289 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚322 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4289 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 4517085184 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚66630680 PIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚67 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17104896 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚209 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚472168529133568 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚538462167138304 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 _runtime‚ 5263.70787716 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚546464378249216 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 5494538240 _step‚1634â $pu2Ž.- _wandbruntime‚5263  _runtime‚5263ØÍÑé.ÉM VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚16653984 trainer/global_step‚19849 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚4295 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚73502516 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 4799641201664 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 5494538240 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚322 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚546490931658752 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚541691248508928 B9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚5264 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚472191576506368 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚61478912 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 4517085184 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 4552982528 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 122905600 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚429 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 845178880 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 125829120 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚546490872605696 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 IBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚60 NADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚58387428 K>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59053056 LCDeviceStatsMonitor.on_train_batch_end/allocation.smalÌMBËæl_pool.current‚4293 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 4789648003597 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚262 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚73502518 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 5368709120 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚62416280 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚45762378 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚66638260 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚66633967 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 4654602240 NGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚82 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 4789689351665 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 5494538240 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 125829120 _runtime‚5263.873974064 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17104896 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 831258624 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 125829120 E6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚322 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚541691231404032 SFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚66776064 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚543278059322865 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚538488369971200 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 5368709120 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚41948160 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚73502518 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚477267859966464 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521206 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚ 140140778 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚73502516 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚41348068 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚262 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚ 140136483 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚ 140136483 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚4293 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 122905600 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚66638260 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚62416364 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚20643840 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17104896 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654602240 B5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5264 DDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 140140778 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚66633967 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚16653902 NADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚59053056 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 4552982528 H@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚665 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 4799641201664 G@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚84 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚45762380 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚209 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚543278000935437 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚322 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚322 I@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚4876 PDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚472191555862528 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚46132224Ö¯éÔ&.ÎM E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚73502516 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚41348068 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚262 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚ 140136483 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚ 140136483 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚4293 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 122905600 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚66638260 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚62416364 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚20643840 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17104896 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654602240 B5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5264 DDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 140140778 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚66633967 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚16653902 NADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚59053056 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 4552982528 H@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚665 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 4799641201664 G@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚84 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚45762380 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚209 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚543278000935437 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚322 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚322 I@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚4876 PDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚472191555862528 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚46132224 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚16653984 trainer/global_step‚19849 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚4295 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚73502516 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 4799641201664 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 5494538240 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚322 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚546490931658752 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚541691248508928 B9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚5264 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚472191576506368 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚61478912 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 4517085184 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 4552982528 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 122905600 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚429 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 845178880 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 125829120 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚546490872605696 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 IBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚60 NADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚58387428 K>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59053056 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚4293 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 4789648003597 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚262 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚73502518 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 5368709120 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚62416280 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚45762378 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚66638260 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚66633967 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 4654602240 NGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚82 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 4789689351665 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 5494538240 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 125829120 _runtime‚5263.873974064 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17104896 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 831258624 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 125829120 E6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚322 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚541691231404032 SFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚66776064 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚543278059322865 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚538488369971200 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 5368709120 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚41948160 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚73502518 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚477267859966464 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521206 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚ 140140778ã ¼G"2‘.- _wandbruntime‚5263  _runtime‚5263ìÈUt’.î 0loss/FRt2/assignment_loss‚0.3958462139776224 trainer/global_step‚19849 % _timestamp‚1.7866639153447173e+09 _step‚1636 0loss/FRt1/assignment_loss‚0.4861082189069177 .loss/FRt1/detection_loss‚0.470497471111728 /loss/FRt2/detection_loss‚0.4924559214403205 &loss/total_loss‚1.8449078254365885 epoch‚49 _runtime‚5263.8745915827Và©y“.ó .loss/FRt1/detection_loss‚0.470497471111728 /loss/FRt2/detection_loss‚0.4924559214403205 &loss/total_loss‚1.8449078254365885 epoch‚49 _runtime‚5263.874591582 0loss/FRt2/assignment_loss‚0.3958462139776224 trainer/global_step‚19849 % _timestamp‚1.7866639153447173e+09 _step‚1636 0loss/FRt1/assignment_loss‚0.4861082189069177ä Å/Š¡”.‚(:– ì§ùÓ˜Ÿ™á gpu.0.gpu‚62.0 gpu.0.memory‚11, gpu.0.memoryAllocated‚7.0313262939453125) gpu.0.memoryAllocatedBytes‚ 6039863296 gpu.0.temp‚36.0 gpu.0.powerWatts‚171.729' gpu.0.enforcedPowerLimitWatts‚500.0) gpu.0.powerPercent‚34.345800000000004 gpu.0.smClock‚1410 gpu.0.memoryClock‚1593$ gpu.0.correctedMemoryErrors‚1955# gpu.0.uncorrectedMemoryErrors‚0>• Í •.‚(:• ì§ùÓèûÛâ$ memory_percent‚68.63934993743896 cpu‚27.26328639723033 disk.sda3.out‚ 96.75390625$ disk.nvme0c0n1.in‚76282.15234375" disk.nvme0n1.in‚76282.15234375 disk.drbd2165.in‚0 disk.drbd2165.out‚0% disk./.usageGB‚487.43780517578125 disk.sda1.out‚0 disk.dm-1.in‚ 47874.4453125 disk.dm-1.out‚ 96.75390625) proc.memory.availableMB‚ 5138.12890625) proc.memory.percent‚14.36917781829834 proc.cpu.threads‚17 network.recv‚ 9.851576e+06* disk./.usagePercent‚32.712360092906565 disk.sda3.in‚ 47874.4453125 disk.sda.in‚ 47874.4453125 disk.sda1.in‚0& disk.nvme0n1.out‚63611.69482421875# disk.dm-0.out‚63611.69482421875# proc.memory.rssMB‚ 2354.24609375 network.sent‚ 2.5770417e+07( disk.nvme0c0n1.out‚63611.69482421875 disk.sda.out‚ 96.75390625 disk.sda2.in‚0 disk.sda2.out‚0 disk.dm-0.in‚76282.15234375W å2–.- _wandbruntime‚5276  _runtime‚5276(B6ÚÈ—. & lr-AdamW/pg1‚9.827705482579052e-08 & lr-AdamW/pg2‚9.827705482579052e-08 trainer/global_step‚19899 $ _timestamp‚1.786663927946891e+09 _runtime‚5276.476919026 _step‚1637ó©þ͘.Ç $ _timestamp‚1.786663927946891e+09 _runtime‚5276.476919026 _step‚1637 & lr-AdamW/pg1‚9.827705482579052e-08 & lr-AdamW/pg2‚9.827705482579052e-08 trainer/global_step‚19899å íû^2™.- _wandbruntime‚5276  _runtime‚52761·äŽÅ'š.¿O [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 4811038843904 JDeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4289 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 _step‚1638 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚41945088 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚61478912 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚262 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚73676613 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17104896 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚73676613 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 4810996898816 KBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚4876 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚209 BDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚547792022854656 PCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚58387380 IBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚85 PCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚59049984 C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚4289 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 4552982528 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚66800069 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚322 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 5494538240 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 5494538240 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚542981025955840 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚16693881 UHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚64681984 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 JBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚665 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚62564221 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚16693798 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 4800979241593 % _timestamp‚1.7866639279478595e+09 _runtime‚5276.481882799 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 125829120 G>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4876 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚73676615 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚44038144 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚544572494444077 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 4517085184 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 5368709120 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚547792022854656 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 F>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚542981043060736 trainer/global_step‚19899 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 122905600 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚322 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 4552982528 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 4654602240 JDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚547792022854656 PCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚58387380 IBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚85 PCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚59049984 C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚4289 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 4552982528 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚66800069 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚322 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 5494538240 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 5494538240 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚542981025955840 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚16693881 UHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚64681984 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 JBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚665 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚62564221 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚16693798 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 4800979241593 % _timestamp‚1.7866639279478595e+09 _runtime‚5276.481882799 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 125829120 G>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4876 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚73676615 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚44038144 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚544572494444077 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 4517085184 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 5368709120 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚547792022854656 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 F>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚542981043060736 trainer/global_step‚19899 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 122905600 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚322 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 4552982528 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 4654602240 JDeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4289 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 _step‚1638 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚41945088 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚61478912 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚262 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚73676613 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17104896 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚73676613 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 4810996898816 KBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚4876 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚209 B6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚322 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 4801251162273 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 5494538240 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚16694659 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 5368709120 K>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59053056 I@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚4876 BDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚62567223 $ _timestamp‚1.786663928116285e+09 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 122905600 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 5494538240 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 125829120 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 122905600 NGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚64 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 5088589803008 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚543007348293632 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚62567289 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚478432534949376 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 4517085184 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 D=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚60 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚4293 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17104896 E8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚262 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 125829120 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚322 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚73680168 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 4654602240 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚16694595 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚539797659648000 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚539797642608640 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 125829120 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚223 >5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5264 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚73680166 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 845178880 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚41937920 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚45872630 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚543007348293632 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 4811269915648 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚66799069 G@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚66 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚429 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 4811227967488 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚209 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚473343924502528 _step‚1639 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚73680166 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 831258624 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚544598910810273 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 5494538240 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚547818635314176 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚41948160 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚41348068 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚41948160 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚473343945146368 NADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚58387428 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚322 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚543007365398528 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 5088547865088 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521206 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚ 140479235 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚66803362 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚ 140483530 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚45872628 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚478432472367616 DDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚262 trainer/global_step‚19899 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654602240 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17104896 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 140483530 B9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚5264 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚73680168 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚322 IBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚60šº8<Ó&ž.ÍM UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚41348068 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚41948160 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚473343945146368 NADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚58387428 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚322 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚543007365398528 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 5088547865088 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521206 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚ 140479235 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚66803362 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚ 140483530 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚45872628 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚478432472367616 DDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚262 trainer/global_step‚19899 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654602240 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17104896 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 140483530 B9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚5264 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚73680168 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚322 IBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚60 _runtime‚5276.646717798 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚544598852422845 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚66799069 P6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚322 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 4801251162273 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 5494538240 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚16694659 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 5368709120 K>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59053056 I@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚4876 BDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚62567223 $ _timestamp‚1.786663928116285e+09 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 122905600 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 5494538240 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 125829120 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 122905600 NGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚64 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 5088589803008 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚543007348293632 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚62567289 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚478432534949376 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 4517085184 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 D=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚60 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚4293 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17104896 E8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚262 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 125829120 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚322 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚73680168 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 4654602240 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚16694595 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚539797659648000 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚539797642608640 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 125829120 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚223 >5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5264 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚73680166 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 845178880 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚41937920 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚45872630 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚543007348293632 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 4811269915648 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚66799069 G@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚66 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚429 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 4811227967488 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚209 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚473343924502528 _step‚1639 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚73680166 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 831258624 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚544598910810273 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 5494538240 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚547818635314176 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚41948160 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0ç ³Ë2Ÿ.- _wandbruntime‚5276  _runtime‚5276›¸ïv .ð 0loss/FRt1/assignment_loss‚0.5022074093915477 /loss/FRt1/detection_loss‚0.5045727483054139 &loss/total_loss‚1.8421530645013033 % _timestamp‚1.7866639281173656e+09 _runtime‚5276.647213084 0loss/FRt2/assignment_loss‚0.3706948095514818 0loss/FRt2/detection_loss‚0.46467809725285975 epoch‚49 trainer/global_step‚19899 _step‚1640ˆA{¡.õ /loss/FRt1/detection_loss‚0.5045727483054139 &loss/total_loss‚1.8421530645013033 % _timestamp‚1.7866639281173656e+09 _runtime‚5276.647213084 0loss/FRt2/assignment_loss‚0.3706948095514818 0loss/FRt2/detection_loss‚0.46467809725285975 epoch‚49 trainer/global_step‚19899 _step‚1640 0loss/FRt1/assignment_loss‚0.5022074093915477è =UõÔ¡¢.‚(:– û§ùÓ§öõà gpu.0.gpu‚61.0 gpu.0.memory‚11, gpu.0.memoryAllocated‚7.0313262939453125) gpu.0.memoryAllocatedBytes‚ 6039863296 gpu.0.temp‚36.0 gpu.0.powerWatts‚130.319' gpu.0.enforcedPowerLimitWatts‚500.0) gpu.0.powerPercent‚26.063799999999997 gpu.0.smClock‚1410 gpu.0.memoryClock‚1593$ gpu.0.correctedMemoryErrors‚1955# gpu.0.uncorrectedMemoryErrors‚0ßZ‡’“£.‚(:ˆ û§ùÓþ’±ã disk.sda.out‚ 96.87890625 disk.sda1.out‚0! disk.nvme0n1.in‚ 76311.1328125 disk.drbd2165.out‚0 disk.sda3.in‚ 48925.7109375 disk.sda3.out‚ 96.87890625) disk./.usagePercent‚32.71267191008327 disk.sda.in‚ 48925.7109375 disk.dm-0.in‚ 76311.1328125# disk.nvme0c0n1.in‚ 76311.1328125% disk.nvme0c0n1.out‚63619.66015625$ memory_percent‚68.64097118377686) proc.memory.availableMB‚ 5137.86328125# proc.memory.rssMB‚ 2354.24609375# disk.nvme0n1.out‚63619.66015625 disk.dm-1.out‚ 96.87890625$ disk./.usageGB‚487.4424514770508 disk.sda1.in‚0 disk.sda2.out‚0) proc.memory.percent‚14.36917781829834 cpu‚27.262771706489808 disk.sda2.in‚0 disk.dm-1.in‚ 48925.7109375 disk.drbd2165.in‚0 disk.dm-0.out‚63619.66015625 proc.cpu.threads‚17 network.sent‚ 2.5863589e+07 network.recv‚ 9.881342e+06ø“Ñu2¤.- _wandbruntime‚5289  _runtime‚5289 UüË¥.Å _step‚1641 ' lr-AdamW/pg1‚2.5058600026661716e-08 ' lr-AdamW/pg2‚2.5058600026661716e-08 trainer/global_step‚19949 % _timestamp‚1.7866639406661072e+09 _runtime‚5289.197897996 îvЦ.Ê _step‚1641 ' lr-AdamW/pg1‚2.5058600026661716e-08 ' lr-AdamW/pg2‚2.5058600026661716e-08 trainer/global_step‚19949 % _timestamp‚1.7866639406661072e+09 _runtime‚5289.197897996é ŒäH?2§.- _wandbruntime‚5289  _runtime‚52895iŸþx¨.¾O L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚66960884 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚41945088 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚4287 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚45980675 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚322 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚73854265 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚66960884 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚16734720 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚61478912 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 122306526 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 5368709120 @7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚5264 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚4287 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚66965171 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚479573843714560 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚54523904 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚545893287544105 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 trainer/global_step‚19949 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚41945088 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.f Ôü¸L reed‚73854263 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚479573918882304 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 125829120 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚544297142845440 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚16734638 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 4517085184 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 125829120 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚ 140819436 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚322 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚541080746491904 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚322 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17104896 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 4822625609728 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚73854265 UHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚75167744 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚262 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4289 BDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚549119726510080 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 122905600 JBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚665 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 % _timestamp‚1.7866639406668272e+09 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚429 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 5494538240 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚62715311 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 845178880 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 _runtime‚ 5289.19799531 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚549119785560064 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚544297142845440 IBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚84 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 5100597944832 PCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚58387380 JDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4876 KDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚60 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚544297159950336 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17104896 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 4628521206 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 4812541052201 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚62715395 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚474473245769728 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚545893345931485 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚322 IBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚60 F>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 PCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚59049984 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚223 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚ 140815147 JDeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4289 BDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚549119726510080 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 122905600 JBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚665 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 % _timestamp‚1.7866639406668272e+09 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚429 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 5494538240 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚62715311 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 845178880 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 _runtime‚ 5289.19799531 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚549119785560064 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚544297142845440 IBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚84 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 5100597944832 PCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚58387380 JDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4876 KDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚60 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚544297159950336 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17104896 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 4628521206 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 4812541052201 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚62715395 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚474473245769728 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚545893345931485 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚322 IBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚60 F>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 PCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚59049984 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚223 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚ 140815147 JDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 125829120ê A2ª.- _wandbruntime‚5289  _runtime‚5289f|ö¾Í&«.ÇM D6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚ 140821987 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654602240 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 5368709120 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 _step‚1643 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚549146279916544 I@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚4876 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚209 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521206 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 125829120 DDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59053056 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚544323482288128 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚262 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 4552982528 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚544323465183232 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚223 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚479597204827648 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 831258624 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚541106932285440 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 5368709120 IBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚60 PDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 140826282 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚66968464 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 B8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚544323465183232 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚73857816 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚45982880 NGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚70 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚474496313786368 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 5494538240 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 5494538240 $ _timestamp‚1.786663940836302e+09 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 4517085184 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17104896 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 5100842811904 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚429 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 5368709120 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚322 E6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚322 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 122306526 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚4293 G@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚72 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚73857816 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚544323482288128 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 4822814733312 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚ 140821987 B9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚5264 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 4812771624813 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚20643840 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚545919703910253 >5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5264 NADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚59053056 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚62718303 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 4812812972881 _runtime‚ 5289.36680707 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚4295 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17104896 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 122905600 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚62718375 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚479597135954432 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚41348068 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚322 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚66968464 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚549146338969600 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 4822856681472 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 4822814733312 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚45982878 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚66964171 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 125829120 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 4654602240 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚41948160 H@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚665 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚16735495 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 5100891041280 EDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚322 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚ 140826282 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚41948160 G@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚60 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 NADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚58387428 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚262 D=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚60 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚549146338969600 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚4293 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 845178880 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 trainer/global_step‚19949wIœAɬ.ÌM SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 122306526 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚4293 G@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚72 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚73857816 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚544323482288128 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 4822814733312 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚ 140821987 B9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚5264 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 4812771624813 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚20643840 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚545919703910253 >5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5264 NADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚59053056 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚62718303 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 4812812972881 _runtime‚ 5289.36680707 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚4295 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17104896 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 122905600 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚62718375 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚479597135954432 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚41348068 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚322 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚66968464 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚549146338969600 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 4822856681472 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 4822814733312 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚45982878 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚66964171 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 125829120 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 4654602240 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚41948160 H@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚665 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚16735495 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 5100891041280 EDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚322 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚ 140826282 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚41948160 G@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚60 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 NADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚58387428 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚262 D=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚60 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚549146338969600 LCDeviceStatsMonið:zW tor.on_train_batch_end/allocation.small_pool.current‚4293 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 845178880 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 trainer/global_step‚19949 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚ 140821987 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654602240 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 5368709120 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 _step‚1643 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚549146279916544 I@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚4876 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚209 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521206 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 125829120 DDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59053056 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚544323482288128 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚262 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 4552982528 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚544323465183232 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚223 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚479597204827648 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 831258624 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚541106932285440 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 5368709120 IBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚60 PDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 140826282 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚66968464 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 B8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚544323465183232 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚73857816 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚45982880 NGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚70 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚474496313786368 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 5494538240 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 5494538240 $ _timestamp‚1.786663940836302e+09 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 4517085184 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17104896 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 5100842811904 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚429 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 5368709120 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚322 E6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚322ë õÑn[2­.- _wandbruntime‚5289  _runtime‚5289³ïE7t®.î .loss/FRt1/detection_loss‚0.510855363583931 /loss/FRt2/detection_loss‚0.4602567874816153 %loss/total_loss‚1.821395246821544 epoch‚49 trainer/global_step‚19949 _runtime‚5289.367295152 % _timestamp‚1.7866639408374367e+09 _step‚1644 1loss/FRt1/assignment_loss‚0.47316019490858824 0loss/FRt2/assignment_loss‚0.3771229008474094a3y¯.ó trainer/global_step‚19949 _runtime‚5289.367295152 % _timestamp‚1.7866639408374367e+09 _step‚1644 1loss/FRt1/assignment_loss‚0.47316019490858824 0loss/FRt2/assignment_loss‚0.3771229008474094 .loss/FRt1/detection_loss‚0.510855363583931 /loss/FRt2/detection_loss‚0.4602567874816153 %loss/total_loss‚1.821395246821544 epoch‚49ì Èiö–°.‚(:‹ Š¨ùÓµ§úà gpu.0.gpu‚64.0 gpu.0.memory‚12, gpu.0.memoryAllocated‚7.0313262939453125) gpu.0.memoryAllocatedBytes‚ 6039863296 gpu.0.temp‚36.0 gpu.0.powerWatts‚217.136' gpu.0.enforcedPowerLimitWatts‚500.0 gpu.0.powerPercent‚43.4272 gpu.0.smClock‚1410 gpu.0.memoryClock‚1593$ gpu.0.correctedMemoryErrors‚1955# gpu.0.uncorrectedMemoryErrors‚013BŒ±.‚(: Š¨ùÓä¬Øâ disk.nvme0n1.in‚ 76319.421875 disk.drbd2165.in‚0 disk.dm-0.in‚ 76319.421875" disk.dm-0.out‚63663.8330078125# proc.memory.availableMB‚5137.75# proc.memory.rssMB‚ 2354.24609375 disk.sda3.out‚ 97.14453125 disk.sda1.out‚0 disk.sda2.in‚0% disk.nvme0n1.out‚63663.8330078125 cpu‚27.262321151584093 network.recv‚ 9.90527e+06% disk./.usageGB‚487.44257736206055 disk.sda.in‚ 48962.953125 disk.sda.out‚ 97.14453125 disk.sda1.in‚0 disk.sda2.out‚0 disk.dm-1.in‚ 48962.953125 disk.dm-1.out‚ 97.14453125 network.sent‚ 2.5931365e+07( disk./.usagePercent‚32.7126803583319" disk.nvme0c0n1.in‚ 76319.421875' disk.nvme0c0n1.out‚63663.8330078125 disk.drbd2165.out‚0$ memory_percent‚68.64166259765625) proc.memory.percent‚14.36917781829834 proc.cpu.threads‚17 disk.sda3.in‚ 48962.953125dr2².- _wandbruntime‚5301  _runtime‚530105Õwɳ.à trainer/global_step‚19999 % _timestamp‚1.7866639532712824e+09 _runtime‚5301.801287271 _step‚1645 & lr-AdamW/pg1‚9.634271214142842e-12 & lr-AdamW/pg2‚9.634271214142842e-12¦Rñlδ.È & lr-AdamW/pg2‚9.634271214142842e-12 trainer/global_step‚19999 % _timestamp‚1.7866639532712824e+09 _runtime‚5301.801287271 _step‚1645 & lr-AdamW/pg1‚9.634271214142842e-12í Ÿ³÷ 2µ.- _wandbruntime‚5301  _runtime‚5301cn|Å'¶.¿O ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 4824102862809 @7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚5264 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚545613276839936 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 4552982528 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚322 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 125829120 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚262 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚ 141157899 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚550447430165504 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚41945088 JDeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4289 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 122306526 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 5494538240 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 125829120 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚262 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚542390053208064 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17104896 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚16775347 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 _runtime‚5301.802475717 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚16775412 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚67130273 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17104896 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 5494538240 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚262 JBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚665 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚547214139031513 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 5112870209024 M@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚59049984 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚550447430165504 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚62866337 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚475625614409728 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚61478912 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 4517085184 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 5494538240 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 % _timestamp‚1.7866639532722266e+09 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 4654602240 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 845178880 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 5112914247168 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚550447489215488 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚74031915 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 trainer/global_step‚19999 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚74031913 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚67125986 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚475625635053568 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚547214197418893 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚46090925 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚550447489215488 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 125829120 F>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚545613276839936 IBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚67 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 5368709120 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 5368709120 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚322 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚545613259735040 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 4834212375552 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚46090923 PCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚59049984 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚4287 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚223 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚322 KDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚60 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 122905600 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚480738484618752 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚67130273 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 4834170430464 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 4834170430464 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚480738549300736 _step‚1646 C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚4289 G>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4876 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚542390036168704 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚322 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 831258624 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚4287 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 4834212375552 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚ 141162188 PCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚58387380 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 4628521206 IBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚60 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚74031913 KBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚4876 UHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚64681984 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚44038144 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 5368709120 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚41945088 D;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚5264 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚209 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚20643840 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 4824144210829 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚41348020 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚67125986 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 4654602240 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 122905600 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚62866270 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0ó”$Ê'·.ÄO WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17104896 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚16775347 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 _runtime‚5301.802475717 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚16775412 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚67130273 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17104896 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 5494538240 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚262 JBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚665 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚547214139031513 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 5112870209024 M@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚59049984 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚550447430165504 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚62866337 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚475625614409728 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚61478912 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 4517085184 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 5494538240 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 % _timestamp‚1.7866639532722266e+09 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 4654602240 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 845178880 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 5112914247168 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚550447489215488 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚74031915 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 trainer/global_step‚19999 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚74031913 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚67125986 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚475625635053568 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚547214197418893 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚46090925 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚550447489215488 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 125829120 F>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚665 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚545613276839936 IBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚67 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 5368709120 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 5368709120 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚322 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚545613259735040 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 4834212375552 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚46090923 PCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚59049984 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚4287 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚223 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚322 KDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚60 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 122905600 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚480738484618752 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚67130273 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 4834170430464 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 4834170430464 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚480738549300736 _step‚1646 C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚4289 G>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚4876 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚542390036168704 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚322 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 831258624 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚4287 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 4834212375552 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚ 141162188 PCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚58387380 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 4628521206 IBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚60 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚74031913 KBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚4876 UHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚64681984 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚44038144 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 5368709120 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚41945088 D;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚5264 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚209 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚20643840 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 4824144210829 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚41348020 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚67125986 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 4654602240 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 122905600 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚62866270 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 4824102862809 @7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚5264 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚545613276839936 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 4552982528 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚322 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 125829120 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚262 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚ 141157899 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚550447430165504 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚41945088 JDeviceStatsMonitor.on_train_batch_start/allocation.all.current‚4289 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 122306526 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 5494538240 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 125829120 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚262 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚542390053208064î G»t2¸.- _wandbruntime‚5301  _runtime‚5301p!¹.ÉM S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚550473983571968 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚16776143 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚547240613785089 K>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59053056 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 4834443447296 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 4834401499136 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚475648682426368 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 4824374783489 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17104896 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚62869271 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 5113159963136 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚322 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚ 141169034 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 4834443447296 NADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚59053056 G@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚77 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚48229376 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 5368709120 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 % _timestamp‚1.7866639534409938e+09 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 4552982528 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 141169034 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 125829120 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 125829120 H@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚665 NGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚75 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 5494538240 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚67129273 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚550474042625024 P6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚322 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚262 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 4654602240 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17104896 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_矙±7pool.freed‚ 4834401499136 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚41948160 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚67129273 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 5368709120 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚67133566 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚209 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚61478912 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚262 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654602240 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚46093128 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚480761773516288 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 >5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5264 B9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚5264 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚16776218 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚475648661782528 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚322 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚545639599177728 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 4552982528 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚ 141164739 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 5494538240 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚20643840 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚4293 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚74035466 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 122905600 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚74035468 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚4293 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 IBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚60 _runtime‚5301.971344359 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 122905600 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚ 141164739 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚480761842389504 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 NADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚58387428 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚41348068 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚4295 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚542416221962240 D=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚60 trainer/global_step‚19999 DDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚545639599177728 EDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521206 G@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚60 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 5368709120 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚74035468 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚74035466 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚429 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚542416239001600 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚545639582072832 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 4517085184 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚262 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚67133566 SFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚68873216³È‰YÔ&º.ÎM NADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚59053056 G@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚77 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚48229376 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 5368709120 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 % _timestamp‚1.7866639534409938e+09 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 4552982528 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 141169034 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 125829120 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 125829120 H@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚665 NGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚75 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 5494538240 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚67129273 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚550474042625024 P6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚322 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚262 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 4654602240 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17104896 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 4834401499136 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚41948160 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚67129273 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 5368709120 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚67133566 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚209 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚61478912 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚262 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 4654602240 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚46093128 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚480761773516288 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 >5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚5264 B9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚5264 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚16776218 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚475648661782528 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚322 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚545639599177728 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 4552982528 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚ 141164739 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 5494538240 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚20643840 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚4293 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚74035466 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 122905600 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚74035468 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚4293 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 IBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚60 _runtime‚5301.971344359 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 122905600 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚ 141164739 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚480761842389504 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 NADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚58387428 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚41348068 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚4295 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚542416221962240 D=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚60 trainer/global_step‚19999 DDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚545639599177728 EDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 4628521206 G@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚60 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 5368709120 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚74035468 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚74035466 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚429 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚542416239001600 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚545639582072832 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 4517085184 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚262 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚67133566 SFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚68873216 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚550473983571968 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚16776143 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚547240613785089 K>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚59053056 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 4834443447296 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 4834401499136 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚475648682426368 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 4824374783489 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17104896 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚62869271 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 5113159963136 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚322 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚ 141169034 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 4834443447296ï i0">2».- _wandbruntime‚5301  _runtime‚5301ÛFÓ’u¼.ï 0loss/FRt2/assignment_loss‚0.3627661546096286 /loss/FRt1/detection_loss‚0.5211706220681651 %loss/total_loss‚1.824146568081622 % _timestamp‚1.7866639534420748e+09 _runtime‚5301.971887386 _step‚1648 1loss/FRt1/assignment_loss‚0.47144636144383406 /loss/FRt2/detection_loss‚0.4687634299599943 epoch‚49 trainer/global_step‚19999GÅàMz½.ô /loss/FRt1/detection_loss‚0.5211706220681651 %loss/total_loss‚1.824146568081622 % _timestamp‚1.7866639534420748e+09 _runtime‚5301.971887386 _step‚1648 1loss/FRt1/assignment_loss‚0.47144636144383406 /loss/FRt2/detection_loss‚0.4687634299599943 epoch‚49 trainer/global_step‚19999 0loss/FRt2/assignment_loss‚0.3627661546096286ð ‘ÉŒ2¾.- _wandbruntime‚5307  _runtime‚5307ø%ü`Þ¿.Ø -Purity/*FRt/FRt_purity‚0.4789000451564789 /Purity/1FRt/event_purity‚0.4676920175552368 +particle/sensitivity‚0.6273260712623596 _step‚1649 /particle/accuracy_2_of_2‚0.6771190762519836 +particle/specificity‚0.6474528908729553 7validation_average_jet_accuracy‚0.31487220525741577 -Purity/1FRt/FRt_purity‚0.4676920175552368 0Purity/2FRt/event_purity‚0.42971357703208923 particle/accuracy_1_of_1‚1 *validation_accuracy‚0.2117970734834671 -Purity/2FRt/FRt_purity‚0.5659474730491638 /particle/accuracy_1_of_2‚0.9223788380622864 trainer/global_step‚19999 % _timestamp‚1.7866639585059226e+09 _runtime‚5307.036004448 *jet/accuracy_1_of_1‚0.3098890781402588 *jet/accuracy_1_of_2‚0.5737141966819763 0Purity/*FRt/event_purity‚0.46551740169525146 2Purity/*FRt/event_proportion‚0.669805645942688 Purity/0FRt/event_purity‚1 3Purity/0FRt/event_proportion‚0.3301943838596344 *jet/accuracy_2_of_2‚0.2117970734834671 'particle/f_score‚0.5531774163246155 epoch‚49 *loss/val_total_loss‚2.5470850063911445 3Purity/1FRt/event_proportion‚0.6297432780265808 2Purity/2FRt/event_proportion‚0.040062315762043 'particle/accuracy‚0.640322744846344EÀŒwãÀ.Ý *loss/val_total_loss‚2.5470850063911445 3Purity/1FRt/event_proportion‚0.6297432780265808 2Purity/2FRt/event_proportion‚0.040062315762043 'particle/accuracy‚0.640322744846344 -Purity/*FRt/FRt_purity‚0.4789000451564789 /Purity/1FRt/event_purity‚0.4676920175552368 +particle/sensitivity‚0.6273260712623596 _step‚1649 /particle/accuracy_2_of_2‚0.6771190762519836 +particle/specificity‚0.6474528908729553 7validation_average_jet_accuracy‚0.31487220525741577 -Purity/1FRt/FRt_purity‚0.4676920175552368 0Purity/2FRt/event_purity‚0.42971357703208923 particle/accuracy_1_of_1‚1 *validation_accuracy‚0.2117970734834671 -Purity/2FRt/FRt_purity‚0.5659474730491638 /particle/accuracy_1_of_2‚0.9223788380622864 trainer/global_step‚19999 % _timestamp‚1.7866639585059226e+09 _runtime‚5307.036004448 *jet/accuracy_1_of_1‚0.3098890781402588 *jet/accuracy_1_of_2‚0.5737141966819763 0Purity/*FRt/event_purity‚0.46551740169525146 2Purity/*FRt/event_proportion‚0.669805645942688 Purity/0FRt/event_purity‚1 3Purity/0FRt/event_proportion‚0.3301943838596344 *jet/accuracy_2_of_2‚0.2117970734834671 'particle/f_score‚0.5531774163246155 epoch‚49ñ N,RÁ. j1qd7654j@ –¨ùÓà»Çô0`Trainer.fit` stopped: `max_epochs=50` reached. ð¬p‰¡Â. j1qd7654jŽ –¨ùÓÀ”§Å|Epoch 49/49 â”â”â”â”â”â”â”â”â”â”â”â”â”â”â”â”â”â”â”â”â” 400/400 0:01:41 • 0:00:00 3.94it/s v_num: 7654Á)[r%Ã. j1qd7654j –¨ùÓ€”¼Å ½2•2Ä.- _wandbruntime‚5307  _runtime‚5307¸ :èÅ.‚(’»)